jonahgao commented on code in PR #10044:
URL:
https://github.com/apache/arrow-datafusion/pull/10044#discussion_r1565107155
##########
datafusion/expr/src/logical_plan/builder.rs:
##########
@@ -1534,44 +1545,50 @@ impl TableSource for LogicalTableSource {
}
/// Create a [`LogicalPlan::Unnest`] plan
-pub fn unnest(input: LogicalPlan, column: Column) -> Result<LogicalPlan> {
- unnest_with_options(input, column, UnnestOptions::new())
+pub fn unnest(input: LogicalPlan, columns: Vec<Column>) -> Result<LogicalPlan>
{
+ unnest_with_options(input, columns, UnnestOptions::new())
}
/// Create a [`LogicalPlan::Unnest`] plan with options
pub fn unnest_with_options(
input: LogicalPlan,
- column: Column,
+ columns: Vec<Column>,
options: UnnestOptions,
) -> Result<LogicalPlan> {
- let (unnest_qualifier, unnest_field) =
- input.schema().qualified_field_from_column(&column)?;
-
// Extract the type of the nested field in the list.
- let unnested_field = match unnest_field.data_type() {
- DataType::List(field)
- | DataType::FixedSizeList(field, _)
- | DataType::LargeList(field) => Arc::new(Field::new(
- unnest_field.name(),
- field.data_type().clone(),
- unnest_field.is_nullable(),
- )),
- _ => {
- // If the unnest field is not a list type return the input plan.
- return Ok(input);
- }
- };
+ let mut unnested_fields: HashMap<usize, _> =
HashMap::with_capacity(columns.len());
+ // Add qualifiers to the columns.
+ let mut qualified_columns = Vec::with_capacity(columns.len());
+ for c in &columns {
+ let index = input.schema().index_of_column(c)?;
+ let (unnest_qualifier, unnest_field) =
input.schema().qualified_field(index);
+ let unnested_field = match unnest_field.data_type() {
+ DataType::List(field)
+ | DataType::FixedSizeList(field, _)
+ | DataType::LargeList(field) => Arc::new(Field::new(
+ unnest_field.name(),
+ field.data_type().clone(),
+ // Unnesting may produce NULLs even if the list is not null.
+ // For example: unnset([1], []) -> 1, null
Review Comment:
> We can do it in another PR!
👌
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
