jayzhan211 commented on code in PR #10044:
URL:
https://github.com/apache/arrow-datafusion/pull/10044#discussion_r1563882878
##########
datafusion/expr/src/logical_plan/builder.rs:
##########
@@ -1534,44 +1545,50 @@ impl TableSource for LogicalTableSource {
}
/// Create a [`LogicalPlan::Unnest`] plan
-pub fn unnest(input: LogicalPlan, column: Column) -> Result<LogicalPlan> {
- unnest_with_options(input, column, UnnestOptions::new())
+pub fn unnest(input: LogicalPlan, columns: Vec<Column>) -> Result<LogicalPlan>
{
+ unnest_with_options(input, columns, UnnestOptions::new())
}
/// Create a [`LogicalPlan::Unnest`] plan with options
pub fn unnest_with_options(
input: LogicalPlan,
- column: Column,
+ columns: Vec<Column>,
options: UnnestOptions,
) -> Result<LogicalPlan> {
- let (unnest_qualifier, unnest_field) =
- input.schema().qualified_field_from_column(&column)?;
-
// Extract the type of the nested field in the list.
- let unnested_field = match unnest_field.data_type() {
- DataType::List(field)
- | DataType::FixedSizeList(field, _)
- | DataType::LargeList(field) => Arc::new(Field::new(
- unnest_field.name(),
- field.data_type().clone(),
- unnest_field.is_nullable(),
- )),
- _ => {
- // If the unnest field is not a list type return the input plan.
- return Ok(input);
- }
- };
+ let mut unnested_fields: HashMap<usize, _> =
HashMap::with_capacity(columns.len());
+ // Add qualifiers to the columns.
+ let mut qualified_columns = Vec::with_capacity(columns.len());
+ for c in &columns {
+ let index = input.schema().index_of_column(c)?;
+ let (unnest_qualifier, unnest_field) =
input.schema().qualified_field(index);
+ let unnested_field = match unnest_field.data_type() {
+ DataType::List(field)
+ | DataType::FixedSizeList(field, _)
+ | DataType::LargeList(field) => Arc::new(Field::new(
+ unnest_field.name(),
+ field.data_type().clone(),
+ // Unnesting may produce NULLs even if the list is not null.
+ // For example: unnset([1], []) -> 1, null
Review Comment:
Although the test does not cover this case, sadly I don't think there is
currently a way to create a list with nullable as false 🤔 Maybe we can write a
simple rust test for this?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
