This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new df2e1e2587 Fix #8507: Non-null sub-field on nullable struct-field has
wrong nullity (#8623)
df2e1e2587 is described below
commit df2e1e2587340c513743b965f9aef301c4a2a859
Author: Marvin Lanhenke <[email protected]>
AuthorDate: Sat Dec 23 13:09:50 2023 +0100
Fix #8507: Non-null sub-field on nullable struct-field has wrong nullity
(#8623)
* added test
* added guard clause
* rename schema fields
* clippy
---------
Co-authored-by: mlanhenke <[email protected]>
---
datafusion/expr/src/expr_schema.rs | 32 ++++++++++++++++++++++++++++++--
1 file changed, 30 insertions(+), 2 deletions(-)
diff --git a/datafusion/expr/src/expr_schema.rs
b/datafusion/expr/src/expr_schema.rs
index e5b0185d90..ba21d09f06 100644
--- a/datafusion/expr/src/expr_schema.rs
+++ b/datafusion/expr/src/expr_schema.rs
@@ -277,6 +277,13 @@ impl ExprSchemable for Expr {
"Wildcard expressions are not valid in a logical query plan"
),
Expr::GetIndexedField(GetIndexedField { expr, field }) => {
+ // If schema is nested, check if parent is nullable
+ // if it is, return early
+ if let Expr::Column(col) = expr.as_ref() {
+ if input_schema.nullable(col)? {
+ return Ok(true);
+ }
+ }
field_for_index(expr, field, input_schema).map(|x|
x.is_nullable())
}
Expr::GroupingSet(_) => {
@@ -411,8 +418,8 @@ pub fn cast_subquery(subquery: Subquery, cast_to_type:
&DataType) -> Result<Subq
mod tests {
use super::*;
use crate::{col, lit};
- use arrow::datatypes::DataType;
- use datafusion_common::{Column, ScalarValue};
+ use arrow::datatypes::{DataType, Fields};
+ use datafusion_common::{Column, ScalarValue, TableReference};
macro_rules! test_is_expr_nullable {
($EXPR_TYPE:ident) => {{
@@ -548,6 +555,27 @@ mod tests {
assert_eq!(&meta, expr.to_field(&schema).unwrap().metadata());
}
+ #[test]
+ fn test_nested_schema_nullability() {
+ let fields = DFField::new(
+ Some(TableReference::Bare {
+ table: "table_name".into(),
+ }),
+ "parent",
+ DataType::Struct(Fields::from(vec![Field::new(
+ "child",
+ DataType::Int64,
+ false,
+ )])),
+ true,
+ );
+
+ let schema = DFSchema::new_with_metadata(vec![fields],
HashMap::new()).unwrap();
+
+ let expr = col("parent").field("child");
+ assert!(expr.nullable(&schema).unwrap());
+ }
+
#[derive(Debug)]
struct MockExprSchema {
nullable: bool,
