jkosh44 commented on PR #14289:
URL: https://github.com/apache/datafusion/pull/14289#issuecomment-2613703184
It looks like a lot of the array functions don't properly handle `NULL`s.
For example (I did not exhaustively test them all),
```
> SELECT array_sort([1,2,3], NULL);
Internal error: could not cast value to
arrow_array::array::byte_array::GenericByteArray<arrow_array::types::GenericStringType<i32>>.
This was likely caused by a bug in DataFusion's code and we would welcome
that you file an bug report in our issue tracker
> SELECT array_resize([1,2,3], NULL, 0);
Internal error: could not cast value to
arrow_array::array::primitive_array::PrimitiveArray<arrow_array::types::Int64Type>.
This was likely caused by a bug in DataFusion's code and we would welcome
that you file an bug report in our issue tracker
> SELECT array_replace([1,2,3], NULL, NULL);
thread 'main' panicked at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/arrow-data-54.0.0/src/transform/mod.rs:418:13:
assertion `left == right` failed: Arrays with inconsistent types passed to
MutableArrayData
left: Int64
right: Null
stack backtrace:
0: rust_begin_unwind
at
/rustc/9fc6b43126469e3858e2fe86cafb4f0fd5068869/library/std/src/panicking.rs:665:5
1: core::panicking::panic_fmt
at
/rustc/9fc6b43126469e3858e2fe86cafb4f0fd5068869/library/core/src/panicking.rs:76:14
2: core::panicking::assert_failed_inner
3: core::panicking::assert_failed
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:373:5
4: arrow_data::transform::MutableArrayData::with_capacities
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/arrow-data-54.0.0/src/transform/mod.rs:418:13
5: datafusion_functions_nested::replace::general_replace
at
/home/joe/Projects/datafusion/datafusion/functions-nested/src/replace.rs:303:23
6: datafusion_functions_nested::replace::array_replace_inner
at
/home/joe/Projects/datafusion/datafusion/functions-nested/src/replace.rs:394:13
7: core::ops::function::Fn::call
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:79:5
8: datafusion_functions_nested::utils::make_scalar_function::{{closure}}
at
/home/joe/Projects/datafusion/datafusion/functions-nested/src/utils.rs:72:22
9: <datafusion_functions_nested::replace::ArrayReplace as
datafusion_expr::udf::ScalarUDFImpl>::invoke_batch
at
/home/joe/Projects/datafusion/datafusion/functions-nested/src/replace.rs:123:9
10: datafusion_expr::udf::ScalarUDFImpl::invoke_with_args
at
/home/joe/Projects/datafusion/datafusion/expr/src/udf.rs:643:9
11: datafusion_expr::udf::ScalarUDF::invoke_with_args
at
/home/joe/Projects/datafusion/datafusion/expr/src/udf.rs:237:9
12: <datafusion_physical_expr::scalar_function::ScalarFunctionExpr as
datafusion_physical_expr_common::physical_expr::PhysicalExpr>::evaluate
at
/home/joe/Projects/datafusion/datafusion/physical-expr/src/scalar_function.rs:195:22
13:
datafusion_optimizer::simplify_expressions::expr_simplifier::ConstEvaluator::evaluate_to_scalar
at
/home/joe/Projects/datafusion/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs:639:29
14:
<datafusion_optimizer::simplify_expressions::expr_simplifier::ConstEvaluator as
datafusion_common::tree_node::TreeNodeRewriter>::f_up
at
/home/joe/Projects/datafusion/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs:529:30
15:
datafusion_common::tree_node::TreeNode::rewrite::{{closure}}::{{closure}}
at
/home/joe/Projects/datafusion/datafusion/common/src/tree_node.rs:183:13
16: datafusion_common::tree_node::Transformed<T>::transform_parent
at
/home/joe/Projects/datafusion/datafusion/common/src/tree_node.rs:763:44
17: datafusion_common::tree_node::TreeNode::rewrite::{{closure}}
at
/home/joe/Projects/datafusion/datafusion/common/src/tree_node.rs:28:9
18: stacker::maybe_grow
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/stacker-0.1.17/src/lib.rs:55:9
19: datafusion_common::tree_node::TreeNode::rewrite
at
/home/joe/Projects/datafusion/datafusion/common/src/tree_node.rs:177:50
20:
datafusion_optimizer::simplify_expressions::expr_simplifier::ExprSimplifier<S>::simplify_with_cycle_count
at
/home/joe/Projects/datafusion/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs:211:17
21:
datafusion_optimizer::simplify_expressions::expr_simplifier::ExprSimplifier<S>::simplify
at
/home/joe/Projects/datafusion/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs:184:12
22:
datafusion_optimizer::simplify_expressions::simplify_exprs::SimplifyExpressions::optimize_internal::{{closure}}
at
/home/joe/Projects/datafusion/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs:127:25
23: core::ops::function::impls::<impl core::ops::function::FnMut<A> for
&mut F>::call_mut
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:294:13
24: <datafusion_expr::expr::Expr as
datafusion_common::tree_node::TreeNodeContainer<datafusion_expr::expr::Expr>>::map_elements
at
/home/joe/Projects/datafusion/datafusion/expr/src/expr.rs:366:9
25: <alloc::vec::Vec<C> as
datafusion_common::tree_node::TreeNodeContainer<T>>::map_elements::{{closure}}
at
/home/joe/Projects/datafusion/datafusion/common/src/tree_node.rs:873:21
26: core::iter::adapters::map::map_try_fold::{{closure}}
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/iter/adapters/map.rs:95:28
27: <alloc::vec::into_iter::IntoIter<T,A> as
core::iter::traits::iterator::Iterator>::try_fold
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/vec/into_iter.rs:346:25
28: <core::iter::adapters::map::Map<I,F> as
core::iter::traits::iterator::Iterator>::try_fold
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/iter/adapters/map.rs:121:9
29: <core::iter::adapters::GenericShunt<I,R> as
core::iter::traits::iterator::Iterator>::try_fold
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/iter/adapters/mod.rs:191:9
30: <I as
alloc::vec::in_place_collect::SpecInPlaceCollect<T,I>>::collect_in_place
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/vec/in_place_collect.rs:380:13
31: alloc::vec::in_place_collect::from_iter_in_place
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/vec/in_place_collect.rs:271:9
32: alloc::vec::in_place_collect::from_iter_in_place{{reify.shim}}
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/vec/in_place_collect.rs:251:1
33: alloc::vec::in_place_collect::<impl
alloc::vec::spec_from_iter::SpecFromIter<T,I> for alloc::vec::Vec<T>>::from_iter
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/vec/in_place_collect.rs:246:9
34: <alloc::vec::Vec<T> as
core::iter::traits::collect::FromIterator<T>>::from_iter
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/vec/mod.rs:3412:9
35: core::iter::traits::iterator::Iterator::collect
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/iter/traits/iterator.rs:1971:9
36: <core::result::Result<V,E> as
core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter::{{closure}}
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/result.rs:1980:51
37: core::iter::adapters::try_process
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/iter/adapters/mod.rs:160:17
38: <core::result::Result<V,E> as
core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/result.rs:1980:9
39: core::iter::traits::iterator::Iterator::collect
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/iter/traits/iterator.rs:1971:9
40: <alloc::vec::Vec<C> as
datafusion_common::tree_node::TreeNodeContainer<T>>::map_elements
at
/home/joe/Projects/datafusion/datafusion/common/src/tree_node.rs:870:9
41: datafusion_expr::logical_plan::tree_node::<impl
datafusion_expr::logical_plan::plan::LogicalPlan>::map_expressions
at
/home/joe/Projects/datafusion/datafusion/expr/src/logical_plan/tree_node.rs:498:19
42:
datafusion_optimizer::simplify_expressions::simplify_exprs::SimplifyExpressions::optimize_internal
at
/home/joe/Projects/datafusion/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs:125:9
43:
<datafusion_optimizer::simplify_expressions::simplify_exprs::SimplifyExpressions
as datafusion_optimizer::optimizer::OptimizerRule>::rewrite
at
/home/joe/Projects/datafusion/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs:73:9
44: datafusion_optimizer::optimizer::optimize_plan_node
at
/home/joe/Projects/datafusion/datafusion/optimizer/src/optimizer.rs:332:16
45: <datafusion_optimizer::optimizer::Rewriter as
datafusion_common::tree_node::TreeNodeRewriter>::f_up
at
/home/joe/Projects/datafusion/datafusion/optimizer/src/optimizer.rs:318:13
46: datafusion_expr::logical_plan::tree_node::<impl
datafusion_expr::logical_plan::plan::LogicalPlan>::rewrite_with_subqueries::{{closure}}::{{closure}}
at
/home/joe/Projects/datafusion/datafusion/expr/src/logical_plan/tree_node.rs:698:17
47: datafusion_common::tree_node::Transformed<T>::transform_parent
at
/home/joe/Projects/datafusion/datafusion/common/src/tree_node.rs:763:44
48: datafusion_expr::logical_plan::tree_node::<impl
datafusion_expr::logical_plan::plan::LogicalPlan>::rewrite_with_subqueries::{{closure}}
at
/home/joe/Projects/datafusion/datafusion/expr/src/logical_plan/tree_node.rs:386:9
49: stacker::maybe_grow
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/stacker-0.1.17/src/lib.rs:55:9
50: datafusion_expr::logical_plan::tree_node::<impl
datafusion_expr::logical_plan::plan::LogicalPlan>::rewrite_with_subqueries
at
/home/joe/Projects/datafusion/datafusion/expr/src/logical_plan/tree_node.rs:690:50
51: datafusion_optimizer::optimizer::Optimizer::optimize
at
/home/joe/Projects/datafusion/datafusion/optimizer/src/optimizer.rs:388:42
52: datafusion::execution::session_state::SessionState::optimize
at
/home/joe/Projects/datafusion/datafusion/core/src/execution/session_state.rs:714:13
53:
datafusion::execution::session_state::SessionState::create_physical_plan::{{closure}}
at
/home/joe/Projects/datafusion/datafusion/core/src/execution/session_state.rs:732:28
54: datafusion::dataframe::DataFrame::create_physical_plan::{{closure}}
at
/home/joe/Projects/datafusion/datafusion/core/src/dataframe/mod.rs:228:61
55: datafusion_cli::exec::exec_and_print::{{closure}}
at ./src/exec.rs:236:55
56: datafusion_cli::exec::exec_from_repl::{{closure}}::{{closure}}
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.43.0/src/macros/select.rs:557:49
57: <core::future::poll_fn::PollFn<F> as
core::future::future::Future>::poll
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/poll_fn.rs:151:9
58: datafusion_cli::exec::exec_from_repl::{{closure}}
at ./src/exec.rs:176:21
59: datafusion_cli::main_inner::{{closure}}
at ./src/main.rs:214:14
60: datafusion_cli::main::{{closure}}
at ./src/main.rs:131:34
61: <core::pin::Pin<P> as core::future::future::Future>::poll
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
62: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.43.0/src/runtime/park.rs:284:63
63: tokio::runtime::coop::with_budget
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.43.0/src/runtime/coop.rs:107:5
64: tokio::runtime::coop::budget
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.43.0/src/runtime/coop.rs:73:5
65: tokio::runtime::park::CachedParkThread::block_on
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.43.0/src/runtime/park.rs:284:31
66: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.43.0/src/runtime/context/blocking.rs:66:9
67:
tokio::runtime::scheduler::multi_thread::MultiThread::block_on::{{closure}}
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.43.0/src/runtime/scheduler/multi_thread/mod.rs:87:13
68: tokio::runtime::context::runtime::enter_runtime
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.43.0/src/runtime/context/runtime.rs:65:16
69: tokio::runtime::scheduler::multi_thread::MultiThread::block_on
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.43.0/src/runtime/scheduler/multi_thread/mod.rs:86:9
70: tokio::runtime::runtime::Runtime::block_on_inner
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.43.0/src/runtime/runtime.rs:370:45
71: tokio::runtime::runtime::Runtime::block_on
at
/home/joe/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.43.0/src/runtime/runtime.rs:340:13
72: datafusion_cli::main
at ./src/main.rs:131:5
73: core::ops::function::FnOnce::call_once
at
/home/joe/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose
backtrace.
killed
Process finished with exit code 9
```
An alternative approach would be to add a function like the following to the
`ScalarUDFImpl` trait
```Rust
/// Returns true if the function should return NULL when any of the
arguments are NULL, false
/// otherwise.
fn propagates_nulls(&self) -> bool;
```
Then we could handle this for all functions in the same place at some higher
level. Maybe somewhere like `make_scalar_function`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
