gabotechs commented on code in PR #27099:
URL: https://github.com/apache/datafusion/pull/27099#discussion_r4602188793


##########
datafusion/substrait/src/logical_plan/consumer/expr/literal.rs:
##########
@@ -573,6 +578,248 @@ pub(crate) fn from_substrait_literal(
     Ok(scalar_value)
 }
 
+/// Renames the fields of the struct literals in `exprs` after those of the 
first
+/// expression in `exprs` that isn't a literal and has a struct type, such as 
a column.
+///
+/// For example, in `coalesce(s, [])` with `s: List<Struct{a, b}>`:
+///
+/// - before: `s` is `List<Struct{a, b}>`, `[]` is `List<Struct{c0, c1}>`
+/// - after: both are `List<Struct{a, b}>`
+///
+/// This is needed because a Substrait literal's type has no field names, so 
the
+/// consumer names its fields `c0`, `c1`, ..., while a column keeps the names 
from its
+/// table. DataFusion matches struct fields by name, so it can't combine
+/// `Struct{a, b}` with `Struct{c0, c1}`. Substrait matches them by position, 
so giving
+/// the literal the column's names doesn't change what the plan means.
+///
+/// A literal whose shape differs from the column's (another number of fields, 
other
+/// nesting) is left as it is, and so is everything if no expression has a 
struct type.
+pub(crate) fn name_struct_literals_like_siblings<'a>(

Review Comment:
   The name of this function does not match what it's doing. Struct literals 
are being named after the first struct expression encountered in the iterator, 
not the first sibling of the provided expressions.
   
   What the function does seems correct though, I think it's just a matter of 
choosing a better name, something like 
`name_struct_literals_like_first_struct_expr`



##########
datafusion/substrait/src/logical_plan/consumer/expr/literal.rs:
##########
@@ -573,6 +578,248 @@ pub(crate) fn from_substrait_literal(
     Ok(scalar_value)
 }
 
+/// Renames the fields of the struct literals in `exprs` after those of the 
first
+/// expression in `exprs` that isn't a literal and has a struct type, such as 
a column.
+///
+/// For example, in `coalesce(s, [])` with `s: List<Struct{a, b}>`:
+///
+/// - before: `s` is `List<Struct{a, b}>`, `[]` is `List<Struct{c0, c1}>`
+/// - after: both are `List<Struct{a, b}>`
+///
+/// This is needed because a Substrait literal's type has no field names, so 
the
+/// consumer names its fields `c0`, `c1`, ..., while a column keeps the names 
from its
+/// table. DataFusion matches struct fields by name, so it can't combine
+/// `Struct{a, b}` with `Struct{c0, c1}`. Substrait matches them by position, 
so giving
+/// the literal the column's names doesn't change what the plan means.
+///
+/// A literal whose shape differs from the column's (another number of fields, 
other
+/// nesting) is left as it is, and so is everything if no expression has a 
struct type.
+pub(crate) fn name_struct_literals_like_siblings<'a>(
+    exprs: impl IntoIterator<Item = &'a mut Expr>,
+    input_schema: &DFSchema,
+) {
+    let mut exprs = exprs.into_iter().collect::<Vec<_>>();
+    // the names come from the first expression that isn't a literal and 
contains a struct
+    let Some(named_type) = exprs
+        .iter()
+        .filter(|e| !is_literal(e) && !matches!(***e, Expr::Lambda(_)))
+        .filter_map(|e| e.get_type(input_schema).ok())
+        .find(contains_struct)
+    else {
+        return;
+    };
+
+    for expr in exprs.iter_mut() {
+        name_struct_literal_like(expr, &named_type);
+    }
+}
+
+/// If `expr` is a struct-typed literal, possibly aliased, gives it the field 
names of
+/// `named_type` when the two have the same shape. A `CASE` whose results are 
all
+/// literals counts as a literal: each result is renamed. Anything else is 
left unchanged.
+pub(crate) fn name_struct_literal_like(expr: &mut Expr, named_type: &DataType) 
{
+    match expr {
+        Expr::Alias(alias) => name_struct_literal_like(&mut alias.expr, 
named_type),
+        Expr::Literal(value, _) => {
+            let value_type = value.data_type();
+            if !contains_struct(&value_type) {
+                return;
+            }
+            let Some(renamed_type) = with_field_names_of(&value_type, 
named_type) else {
+                return;
+            };
+            // if renaming fails, keep the literal: coercion reports the 
mismatch as before
+            if let Ok(renamed) = with_data_type(value, &renamed_type) {
+                *value = renamed;
+            }
+        }
+        Expr::Case(case) if case_results(case).all(is_literal) => {
+            for (_, then) in case.when_then_expr.iter_mut() {
+                name_struct_literal_like(then, named_type);
+            }
+            if let Some(else_expr) = case.else_expr.as_mut() {
+                name_struct_literal_like(else_expr, named_type);
+            }
+        }
+        _ => {}
+    }
+}
+
+/// Returns true if `expr` is a literal, possibly aliased, or a `CASE` whose 
results are
+/// all literals.
+pub(crate) fn is_literal(expr: &Expr) -> bool {
+    match expr {
+        Expr::Alias(alias) => is_literal(&alias.expr),
+        Expr::Literal(..) => true,
+        Expr::Case(case) => case_results(case).all(is_literal),
+        _ => false,
+    }
+}
+
+/// Returns true if `expr` is a literal whose type contains a struct, possibly 
aliased,
+/// or a `CASE` whose results are all literals and include one.
+pub(crate) fn is_struct_literal(expr: &Expr) -> bool {
+    match expr {
+        Expr::Alias(alias) => is_struct_literal(&alias.expr),
+        Expr::Literal(value, _) => contains_struct(&value.data_type()),
+        Expr::Case(case) => {
+            case_results(case).all(is_literal)
+                && case_results(case).any(is_struct_literal)
+        }
+        _ => false,
+    }
+}
+
+/// The results of `case`: its `THEN`s, then its `ELSE` if it has one. Its 
conditions
+/// aren't results: `CASE WHEN x > 0 THEN {c0: 1} ELSE {c0: 2} END` has only 
literal
+/// results.
+fn case_results(case: &Case) -> impl Iterator<Item = &Expr> {
+    case.when_then_expr
+        .iter()
+        .map(|(_, then)| then.as_ref())
+        .chain(case.else_expr.as_deref())
+}
+
+pub(crate) fn contains_struct(data_type: &DataType) -> bool {
+    match data_type {
+        DataType::Struct(_) => true,
+        DataType::List(f)
+        | DataType::LargeList(f)

Review Comment:
   :thinking: Couldn't `ListView` aso contain structs?



##########
datafusion/substrait/tests/cases/consumer_integration.rs:
##########
@@ -751,6 +751,25 @@ mod tests {
         Ok(())
     }
 
+    // Substrait literals and lambda parameters carry no struct field names. 
Reading
+    // fields inside a lambda over `coalesce(zones, [])` needs the empty list 
to take the
+    // column's field names, and the lambda parameter to take the list's.
+    #[tokio::test]
+    async fn test_transform_coalesce_struct_list_with_empty_list() -> 
Result<()> {

Review Comment:
   Got Claude to find an alternative to provide the same coverage without 
needing to commit a new big .json, and being consistent with the other tests 
contributed in this PR:
   
   ```rust
   #[tokio::test]
   async fn roundtrip_struct_literal_next_to_nested_struct_column() -> 
Result<()> {
       let ctx = create_context().await?;
       ctx.sql(
           "CREATE TABLE zones AS SELECT [named_struct('subnet', 
named_struct('id', 'subnet-1'), \
            'zone_name', 'zone-a')] AS z UNION ALL SELECT NULL AS z",
       )
       .await?
       .collect()
       .await?;
       roundtrip_with_ctx("SELECT coalesce(z, []) FROM zones", ctx).await?;
       Ok(())
   }
   ```
   
   



##########
datafusion/substrait/tests/cases/roundtrip_logical_plan.rs:
##########
@@ -2755,6 +2755,466 @@ async fn 
roundtrip_lambda_parameter_struct_field_names() -> Result<()> {
     Ok(())
 }
 
+// Substrait literals carry no struct field names, so a struct literal 
combined with a
+// struct column must take the column's field names for the two to have the 
same type.
+
+/// A context with a table `structs` whose column `s` is a list of structs.
+async fn struct_lists_ctx() -> Result<SessionContext> {
+    let ctx = create_context().await?;
+    ctx.sql(
+        "CREATE TABLE structs AS SELECT [named_struct('a', 1, 'b', 'x')] AS s \
+         UNION ALL SELECT NULL AS s",
+    )
+    .await?
+    .collect()
+    .await?;
+    Ok(ctx)
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_coalesce() -> Result<()> {
+    // coalesce is simplified to a CASE whose ELSE branch is the literal
+    roundtrip_with_ctx(
+        "SELECT coalesce(s, [named_struct('a', 0, 'b', 'z')]) FROM structs",
+        struct_lists_ctx().await?,
+    )
+    .await?;
+    Ok(())
+}
+
+/// A NULL struct in the ELSE branch, from 
https://github.com/apache/datafusion/issues/14632
+#[tokio::test]
+async fn roundtrip_struct_literal_in_case_else() -> Result<()> {
+    roundtrip(
+        "SELECT (CASE WHEN column1 IS NOT NULL THEN named_struct('f1', 
column1) ELSE NULL END) AS res \
+         FROM (VALUES ('a'), (NULL))",
+    )
+    .await
+}
+
+/// Checks a set operation with a NULL struct in one input. The consumer 
rebuilds a union
+/// with the alias above it rather than in each input, so the plans can't be 
compared as
+/// text: this checks the output type and that the plan runs instead.
+async fn assert_union_with_null_struct_roundtrips(sql: &str) -> Result<()> {
+    let ctx = struct_rows_ctx().await?;
+    let plan = ctx.sql(sql).await?.into_optimized_plan()?;
+    let plan2 = substrait_roundtrip(&plan, &ctx).await?;
+    assert_eq!(
+        plan.schema().field(0).data_type(),
+        plan2.schema().field(0).data_type()
+    );
+    DataFrame::new(ctx.state(), plan2).collect().await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_union_with_null_second_input() -> Result<()> 
{
+    assert_union_with_null_struct_roundtrips(
+        "SELECT st AS r FROM struct_rows UNION ALL SELECT NULL AS r FROM 
struct_rows",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_union_with_null_first_input() -> Result<()> {
+    assert_union_with_null_struct_roundtrips(
+        "SELECT NULL AS r FROM struct_rows UNION ALL SELECT st AS r FROM 
struct_rows",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_list() -> Result<()> {
+    roundtrip_with_ctx(
+        "SELECT st IN (named_struct('a', 1, 'b', 'x'), named_struct('a', 2, 
'b', 'y')) \
+         FROM struct_rows",
+        struct_rows_ctx().await?,
+    )
+    .await?;
+    Ok(())
+}
+
+// Keep these plans unoptimized before serialization so the consumer sees the
+// literal and relation shapes emitted by another Substrait producer.
+#[tokio::test]
+async fn roundtrip_struct_literal_dictionary_values() -> Result<()> {
+    use datafusion::arrow::record_batch::RecordBatch;
+    use datafusion::functions::core::expr_fn::coalesce;
+
+    let ctx = SessionContext::new();
+    let dictionary_type = DataType::Dictionary(
+        Box::new(DataType::Int32),
+        Box::new(DataType::Struct(
+            vec![Field::new("a", DataType::Int32, true)].into(),
+        )),
+    );
+    let empty = ScalarValue::List(ScalarValue::new_list_nullable(&[], 
&dictionary_type));
+    // An empty list can carry a dictionary-of-struct type without requiring
+    // support for serializing non-null dictionary literals.
+    ctx.register_batch(
+        "dicts",
+        RecordBatch::try_from_iter(vec![("s", empty.to_array()?)])?,
+    )?;
+    let plan = ctx
+        .table("dicts")
+        .await?
+        .select(vec![coalesce(vec![col("s"), lit(empty)]).alias("result")])?
+        .into_unoptimized_plan();
+    let expected = DataFrame::new(ctx.state(), plan.clone()).collect().await?;
+    let mut proto = to_substrait_plan(&plan, &ctx.state())?;
+    // Supply a complete NamedStruct, as an external producer would: our 
producer
+    // currently omits names nested inside dictionary values.
+    let Some(plan_rel::RelType::Root(root)) = &mut proto.relations[0].rel_type 
else {
+        panic!("expected root");
+    };
+    root.names.push("a".into());
+    let Some(RelType::Project(project)) =
+        &mut root.input.as_mut().expect("root input").rel_type
+    else {
+        panic!("expected project");
+    };
+    let Some(RelType::Read(read)) =
+        &mut project.input.as_mut().expect("project input").rel_type
+    else {
+        panic!("expected read");
+    };
+    read.base_schema
+        .as_mut()
+        .expect("read schema")
+        .names
+        .push("a".into());
+    let restored = from_substrait_plan(&ctx.state(), &proto).await?;
+    let restored = ctx.state().optimize(&restored)?;
+    assert_eq!(
+        plan.schema().field(0).data_type(),
+        restored.schema().field(0).data_type()
+    );
+    assert_eq!(
+        expected,
+        DataFrame::new(ctx.state(), restored).collect().await?
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_independent_function_arguments() -> 
Result<()> {

Review Comment:
   This test also passes on `main`, and it's inconsistent with the shape of the 
other roundtrip tests.
   
   Unless you see this somehow provides any coverage, I'd just remove it.
   
   In general, I'd aim for all the tests committed to this file to follow the 
following pattern:
   
   ```rust
   #[tokio::test]
   async fn roundtrip_struct_literal_union_with_null_first_input() -> 
Result<()> {
       assert_union_with_null_struct_roundtrips(
           "SELECT NULL AS r FROM struct_rows UNION ALL SELECT st AS r FROM 
struct_rows",
       )
       .await
   }
   ```
   
   Ideally, any new test contributed to this file should not be more 
complicated than this.



##########
datafusion/substrait/tests/cases/roundtrip_logical_plan.rs:
##########
@@ -2755,6 +2755,466 @@ async fn 
roundtrip_lambda_parameter_struct_field_names() -> Result<()> {
     Ok(())
 }
 
+// Substrait literals carry no struct field names, so a struct literal 
combined with a
+// struct column must take the column's field names for the two to have the 
same type.
+
+/// A context with a table `structs` whose column `s` is a list of structs.
+async fn struct_lists_ctx() -> Result<SessionContext> {
+    let ctx = create_context().await?;
+    ctx.sql(
+        "CREATE TABLE structs AS SELECT [named_struct('a', 1, 'b', 'x')] AS s \
+         UNION ALL SELECT NULL AS s",
+    )
+    .await?
+    .collect()
+    .await?;
+    Ok(ctx)
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_coalesce() -> Result<()> {
+    // coalesce is simplified to a CASE whose ELSE branch is the literal
+    roundtrip_with_ctx(
+        "SELECT coalesce(s, [named_struct('a', 0, 'b', 'z')]) FROM structs",
+        struct_lists_ctx().await?,
+    )
+    .await?;
+    Ok(())
+}
+
+/// A NULL struct in the ELSE branch, from 
https://github.com/apache/datafusion/issues/14632
+#[tokio::test]
+async fn roundtrip_struct_literal_in_case_else() -> Result<()> {
+    roundtrip(
+        "SELECT (CASE WHEN column1 IS NOT NULL THEN named_struct('f1', 
column1) ELSE NULL END) AS res \
+         FROM (VALUES ('a'), (NULL))",
+    )
+    .await
+}
+
+/// Checks a set operation with a NULL struct in one input. The consumer 
rebuilds a union
+/// with the alias above it rather than in each input, so the plans can't be 
compared as
+/// text: this checks the output type and that the plan runs instead.
+async fn assert_union_with_null_struct_roundtrips(sql: &str) -> Result<()> {
+    let ctx = struct_rows_ctx().await?;
+    let plan = ctx.sql(sql).await?.into_optimized_plan()?;
+    let plan2 = substrait_roundtrip(&plan, &ctx).await?;
+    assert_eq!(
+        plan.schema().field(0).data_type(),
+        plan2.schema().field(0).data_type()
+    );
+    DataFrame::new(ctx.state(), plan2).collect().await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_union_with_null_second_input() -> Result<()> 
{
+    assert_union_with_null_struct_roundtrips(
+        "SELECT st AS r FROM struct_rows UNION ALL SELECT NULL AS r FROM 
struct_rows",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_union_with_null_first_input() -> Result<()> {
+    assert_union_with_null_struct_roundtrips(
+        "SELECT NULL AS r FROM struct_rows UNION ALL SELECT st AS r FROM 
struct_rows",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_list() -> Result<()> {
+    roundtrip_with_ctx(
+        "SELECT st IN (named_struct('a', 1, 'b', 'x'), named_struct('a', 2, 
'b', 'y')) \
+         FROM struct_rows",
+        struct_rows_ctx().await?,
+    )
+    .await?;
+    Ok(())
+}
+
+// Keep these plans unoptimized before serialization so the consumer sees the
+// literal and relation shapes emitted by another Substrait producer.
+#[tokio::test]
+async fn roundtrip_struct_literal_dictionary_values() -> Result<()> {
+    use datafusion::arrow::record_batch::RecordBatch;
+    use datafusion::functions::core::expr_fn::coalesce;
+
+    let ctx = SessionContext::new();
+    let dictionary_type = DataType::Dictionary(
+        Box::new(DataType::Int32),
+        Box::new(DataType::Struct(
+            vec![Field::new("a", DataType::Int32, true)].into(),
+        )),
+    );
+    let empty = ScalarValue::List(ScalarValue::new_list_nullable(&[], 
&dictionary_type));
+    // An empty list can carry a dictionary-of-struct type without requiring
+    // support for serializing non-null dictionary literals.
+    ctx.register_batch(
+        "dicts",
+        RecordBatch::try_from_iter(vec![("s", empty.to_array()?)])?,
+    )?;
+    let plan = ctx
+        .table("dicts")
+        .await?
+        .select(vec![coalesce(vec![col("s"), lit(empty)]).alias("result")])?
+        .into_unoptimized_plan();
+    let expected = DataFrame::new(ctx.state(), plan.clone()).collect().await?;
+    let mut proto = to_substrait_plan(&plan, &ctx.state())?;
+    // Supply a complete NamedStruct, as an external producer would: our 
producer
+    // currently omits names nested inside dictionary values.
+    let Some(plan_rel::RelType::Root(root)) = &mut proto.relations[0].rel_type 
else {
+        panic!("expected root");
+    };
+    root.names.push("a".into());
+    let Some(RelType::Project(project)) =
+        &mut root.input.as_mut().expect("root input").rel_type
+    else {
+        panic!("expected project");
+    };
+    let Some(RelType::Read(read)) =
+        &mut project.input.as_mut().expect("project input").rel_type
+    else {
+        panic!("expected read");
+    };
+    read.base_schema
+        .as_mut()
+        .expect("read schema")
+        .names
+        .push("a".into());
+    let restored = from_substrait_plan(&ctx.state(), &proto).await?;
+    let restored = ctx.state().optimize(&restored)?;
+    assert_eq!(
+        plan.schema().field(0).data_type(),
+        restored.schema().field(0).data_type()
+    );
+    assert_eq!(
+        expected,
+        DataFrame::new(ctx.state(), restored).collect().await?
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_independent_function_arguments() -> 
Result<()> {

Review Comment:
   If a simple roundtrip is not enough to catch some of the bugs being solved 
in this PR, then it means that a roundtrip test is probably not the best way of 
covering it.



##########
datafusion/substrait/src/logical_plan/consumer/expr/literal.rs:
##########
@@ -573,6 +578,248 @@ pub(crate) fn from_substrait_literal(
     Ok(scalar_value)
 }
 
+/// Renames the fields of the struct literals in `exprs` after those of the 
first
+/// expression in `exprs` that isn't a literal and has a struct type, such as 
a column.
+///
+/// For example, in `coalesce(s, [])` with `s: List<Struct{a, b}>`:
+///
+/// - before: `s` is `List<Struct{a, b}>`, `[]` is `List<Struct{c0, c1}>`
+/// - after: both are `List<Struct{a, b}>`
+///
+/// This is needed because a Substrait literal's type has no field names, so 
the
+/// consumer names its fields `c0`, `c1`, ..., while a column keeps the names 
from its
+/// table. DataFusion matches struct fields by name, so it can't combine
+/// `Struct{a, b}` with `Struct{c0, c1}`. Substrait matches them by position, 
so giving
+/// the literal the column's names doesn't change what the plan means.
+///
+/// A literal whose shape differs from the column's (another number of fields, 
other
+/// nesting) is left as it is, and so is everything if no expression has a 
struct type.
+pub(crate) fn name_struct_literals_like_siblings<'a>(
+    exprs: impl IntoIterator<Item = &'a mut Expr>,
+    input_schema: &DFSchema,
+) {
+    let mut exprs = exprs.into_iter().collect::<Vec<_>>();
+    // the names come from the first expression that isn't a literal and 
contains a struct
+    let Some(named_type) = exprs
+        .iter()
+        .filter(|e| !is_literal(e) && !matches!(***e, Expr::Lambda(_)))
+        .filter_map(|e| e.get_type(input_schema).ok())
+        .find(contains_struct)
+    else {
+        return;
+    };
+
+    for expr in exprs.iter_mut() {
+        name_struct_literal_like(expr, &named_type);
+    }
+}
+
+/// If `expr` is a struct-typed literal, possibly aliased, gives it the field 
names of
+/// `named_type` when the two have the same shape. A `CASE` whose results are 
all
+/// literals counts as a literal: each result is renamed. Anything else is 
left unchanged.
+pub(crate) fn name_struct_literal_like(expr: &mut Expr, named_type: &DataType) 
{
+    match expr {
+        Expr::Alias(alias) => name_struct_literal_like(&mut alias.expr, 
named_type),
+        Expr::Literal(value, _) => {
+            let value_type = value.data_type();
+            if !contains_struct(&value_type) {
+                return;
+            }
+            let Some(renamed_type) = with_field_names_of(&value_type, 
named_type) else {
+                return;
+            };
+            // if renaming fails, keep the literal: coercion reports the 
mismatch as before
+            if let Ok(renamed) = with_data_type(value, &renamed_type) {
+                *value = renamed;
+            }
+        }
+        Expr::Case(case) if case_results(case).all(is_literal) => {
+            for (_, then) in case.when_then_expr.iter_mut() {
+                name_struct_literal_like(then, named_type);
+            }
+            if let Some(else_expr) = case.else_expr.as_mut() {
+                name_struct_literal_like(else_expr, named_type);
+            }
+        }
+        _ => {}
+    }
+}
+
+/// Returns true if `expr` is a literal, possibly aliased, or a `CASE` whose 
results are
+/// all literals.
+pub(crate) fn is_literal(expr: &Expr) -> bool {
+    match expr {
+        Expr::Alias(alias) => is_literal(&alias.expr),

Review Comment:
   This is a bit weird. An `Alias` is not a literal, it's an alias, this 
function does not seem to be searching for literals.
   
   For example, a `Case` expression might switch on a statement containing 
references to columns or arbitrary non-literal expressions.
   
   As this function is public to the full crate, I'd try to rename it to 
something more descriptive.



##########
datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs:
##########
@@ -76,26 +79,75 @@ pub async fn from_scalar_function(
             Ok(expr.resolve_lambda_variables(input_schema)?.data)
         }
     } else if let Ok(func) = udf_func {
-        Ok(Expr::ScalarFunction(expr::ScalarFunction::new_udf(
-            func.to_owned(),
-            args,
-        )))
+        build_naming_struct_literals_if_needed(args, input_schema, |args| {
+            Ok(Expr::ScalarFunction(expr::ScalarFunction::new_udf(
+                func.to_owned(),
+                args,
+            )))
+        })
     } else if let Some(op) = name_to_op(fn_name) {
         if args.len() < 2 {
             return not_impl_err!(
                 "Expect at least two arguments for binary operator {op:?}, the 
provided number of operators is {:?}",
                 f.arguments.len()
             );
         }
+        // an operator always combines its operands, so they need the same 
field names
+        name_struct_literals_like_siblings(
+            &mut args.iter_mut().collect::<Vec<_>>(),
+            input_schema,
+        );
         // In those cases we build a balanced tree of BinaryExprs
         arg_list_to_binary_op_tree(op, args)
     } else if let Some(builder) = BuiltinExprBuilder::try_from_name(fn_name) {
+        // these combine their arguments too, as in `s BETWEEN low AND high`
+        name_struct_literals_like_siblings(
+            &mut args.iter_mut().collect::<Vec<_>>(),
+            input_schema,
+        );
         builder.build(consumer, f, args)
     } else {
         not_impl_err!("Unsupported function name: {fn_name:?}")
     }
 }
 
+/// Builds a function call with `build`, renaming its struct literals after a 
same-shaped
+/// argument only if the call doesn't type-check without it.
+///
+/// `coalesce(s, [])` needs one type for `s: List<Struct{a, b}>` and the 
literal's
+/// `List<Struct{c0, c1}>`, so the literal takes `s`'s names. 
`named_struct('left', s,
+/// 'right', {c0: 42})` type-checks as is, and renaming would break a later
+/// `get_field(..., 'c0')`, so the literal is left alone. Function signatures 
can't tell
+/// these apart, hence the check; operators always combine their operands and 
always
+/// rename.
+fn build_naming_struct_literals_if_needed(

Review Comment:
   :see_no_evil: this is pretty hacky... I wish there was a cleaner way of 
doing this.



##########
datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs:
##########
@@ -80,26 +82,66 @@ pub async fn from_scalar_function(
             Ok(expr.resolve_lambda_variables(input_schema)?.data)
         }
     } else if let Ok(func) = udf_func {
-        Ok(Expr::ScalarFunction(expr::ScalarFunction::new_udf(
-            func.to_owned(),
-            args,
-        )))
+        build_naming_struct_literals_if_needed(args, input_schema, |args| {
+            Ok(Expr::ScalarFunction(expr::ScalarFunction::new_udf(
+                func.to_owned(),
+                args,
+            )))
+        })
     } else if let Some(op) = name_to_op(fn_name) {
         if args.len() < 2 {
             return not_impl_err!(
                 "Expect at least two arguments for binary operator {op:?}, the 
provided number of operators is {:?}",
                 f.arguments.len()
             );
         }
+        // an operator always combines its operands, so they need the same 
field names
+        name_struct_literals_like_siblings(&mut args, input_schema);
         // In those cases we build a balanced tree of BinaryExprs
         arg_list_to_binary_op_tree(op, args)
     } else if let Some(builder) = BuiltinExprBuilder::try_from_name(fn_name) {
+        // these combine their arguments too, as in `s BETWEEN low AND high`
+        name_struct_literals_like_siblings(&mut args, input_schema);
         builder.build(consumer, f, args)
     } else {
         not_impl_err!("Unsupported function name: {fn_name:?}")
     }
 }
 
+/// Builds a function call with `build`, renaming its struct literals after a 
same-shaped
+/// argument only if the call doesn't type-check without it.
+///
+/// `coalesce(s, [])` needs one type for `s: List<Struct{a, b}>` and the 
literal's
+/// `List<Struct{c0, c1}>`, so the literal takes `s`'s names. 
`named_struct('left', s,
+/// 'right', {c0: 42})` type-checks as is, and renaming would break a later
+/// `get_field(..., 'c0')`, so the literal is left alone. Function signatures 
can't tell
+/// these apart, hence the check; operators always combine their operands and 
always
+/// rename.
+fn build_naming_struct_literals_if_needed(
+    args: Vec<Expr>,
+    input_schema: &DFSchema,
+    build: impl Fn(Vec<Expr>) -> Result<Expr>,

Review Comment:
   Having a function that takes a function argument and calls it inside 
multiple times sounds pretty convoluted. It feels a bit like hiding dirt under 
the carpet.
   
   



##########
datafusion/substrait/src/logical_plan/consumer/expr/literal.rs:
##########
@@ -573,6 +578,248 @@ pub(crate) fn from_substrait_literal(
     Ok(scalar_value)
 }
 
+/// Renames the fields of the struct literals in `exprs` after those of the 
first
+/// expression in `exprs` that isn't a literal and has a struct type, such as 
a column.
+///
+/// For example, in `coalesce(s, [])` with `s: List<Struct{a, b}>`:
+///
+/// - before: `s` is `List<Struct{a, b}>`, `[]` is `List<Struct{c0, c1}>`
+/// - after: both are `List<Struct{a, b}>`
+///
+/// This is needed because a Substrait literal's type has no field names, so 
the
+/// consumer names its fields `c0`, `c1`, ..., while a column keeps the names 
from its
+/// table. DataFusion matches struct fields by name, so it can't combine
+/// `Struct{a, b}` with `Struct{c0, c1}`. Substrait matches them by position, 
so giving
+/// the literal the column's names doesn't change what the plan means.
+///
+/// A literal whose shape differs from the column's (another number of fields, 
other
+/// nesting) is left as it is, and so is everything if no expression has a 
struct type.
+pub(crate) fn name_struct_literals_like_siblings<'a>(
+    exprs: impl IntoIterator<Item = &'a mut Expr>,
+    input_schema: &DFSchema,
+) {
+    let mut exprs = exprs.into_iter().collect::<Vec<_>>();
+    // the names come from the first expression that isn't a literal and 
contains a struct
+    let Some(named_type) = exprs
+        .iter()
+        .filter(|e| !is_literal(e) && !matches!(***e, Expr::Lambda(_)))
+        .filter_map(|e| e.get_type(input_schema).ok())
+        .find(contains_struct)
+    else {
+        return;
+    };
+
+    for expr in exprs.iter_mut() {
+        name_struct_literal_like(expr, &named_type);
+    }
+}
+
+/// If `expr` is a struct-typed literal, possibly aliased, gives it the field 
names of
+/// `named_type` when the two have the same shape. A `CASE` whose results are 
all
+/// literals counts as a literal: each result is renamed. Anything else is 
left unchanged.
+pub(crate) fn name_struct_literal_like(expr: &mut Expr, named_type: &DataType) 
{
+    match expr {
+        Expr::Alias(alias) => name_struct_literal_like(&mut alias.expr, 
named_type),
+        Expr::Literal(value, _) => {
+            let value_type = value.data_type();
+            if !contains_struct(&value_type) {
+                return;
+            }
+            let Some(renamed_type) = with_field_names_of(&value_type, 
named_type) else {
+                return;
+            };
+            // if renaming fails, keep the literal: coercion reports the 
mismatch as before
+            if let Ok(renamed) = with_data_type(value, &renamed_type) {
+                *value = renamed;
+            }
+        }
+        Expr::Case(case) if case_results(case).all(is_literal) => {
+            for (_, then) in case.when_then_expr.iter_mut() {
+                name_struct_literal_like(then, named_type);
+            }
+            if let Some(else_expr) = case.else_expr.as_mut() {
+                name_struct_literal_like(else_expr, named_type);
+            }
+        }
+        _ => {}
+    }
+}
+
+/// Returns true if `expr` is a literal, possibly aliased, or a `CASE` whose 
results are
+/// all literals.
+pub(crate) fn is_literal(expr: &Expr) -> bool {
+    match expr {
+        Expr::Alias(alias) => is_literal(&alias.expr),
+        Expr::Literal(..) => true,
+        Expr::Case(case) => case_results(case).all(is_literal),
+        _ => false,
+    }
+}
+
+/// Returns true if `expr` is a literal whose type contains a struct, possibly 
aliased,
+/// or a `CASE` whose results are all literals and include one.
+pub(crate) fn is_struct_literal(expr: &Expr) -> bool {

Review Comment:
   Same as above



##########
datafusion/substrait/tests/cases/roundtrip_logical_plan.rs:
##########
@@ -2755,6 +2755,466 @@ async fn 
roundtrip_lambda_parameter_struct_field_names() -> Result<()> {
     Ok(())
 }
 
+// Substrait literals carry no struct field names, so a struct literal 
combined with a
+// struct column must take the column's field names for the two to have the 
same type.
+
+/// A context with a table `structs` whose column `s` is a list of structs.
+async fn struct_lists_ctx() -> Result<SessionContext> {

Review Comment:
   This helper is only used in one place. I don't think we gain much from 
extracting it out, and it's easy to confuse it with another test entry.
   
   I'd just inline it where it's used.
   



##########
datafusion/substrait/tests/cases/roundtrip_logical_plan.rs:
##########
@@ -2755,6 +2755,466 @@ async fn 
roundtrip_lambda_parameter_struct_field_names() -> Result<()> {
     Ok(())
 }
 
+// Substrait literals carry no struct field names, so a struct literal 
combined with a
+// struct column must take the column's field names for the two to have the 
same type.
+
+/// A context with a table `structs` whose column `s` is a list of structs.
+async fn struct_lists_ctx() -> Result<SessionContext> {
+    let ctx = create_context().await?;
+    ctx.sql(
+        "CREATE TABLE structs AS SELECT [named_struct('a', 1, 'b', 'x')] AS s \
+         UNION ALL SELECT NULL AS s",
+    )
+    .await?
+    .collect()
+    .await?;
+    Ok(ctx)
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_coalesce() -> Result<()> {
+    // coalesce is simplified to a CASE whose ELSE branch is the literal
+    roundtrip_with_ctx(
+        "SELECT coalesce(s, [named_struct('a', 0, 'b', 'z')]) FROM structs",
+        struct_lists_ctx().await?,
+    )
+    .await?;
+    Ok(())
+}
+
+/// A NULL struct in the ELSE branch, from 
https://github.com/apache/datafusion/issues/14632
+#[tokio::test]
+async fn roundtrip_struct_literal_in_case_else() -> Result<()> {
+    roundtrip(
+        "SELECT (CASE WHEN column1 IS NOT NULL THEN named_struct('f1', 
column1) ELSE NULL END) AS res \
+         FROM (VALUES ('a'), (NULL))",
+    )
+    .await
+}
+
+/// Checks a set operation with a NULL struct in one input. The consumer 
rebuilds a union
+/// with the alias above it rather than in each input, so the plans can't be 
compared as
+/// text: this checks the output type and that the plan runs instead.
+async fn assert_union_with_null_struct_roundtrips(sql: &str) -> Result<()> {
+    let ctx = struct_rows_ctx().await?;
+    let plan = ctx.sql(sql).await?.into_optimized_plan()?;
+    let plan2 = substrait_roundtrip(&plan, &ctx).await?;
+    assert_eq!(
+        plan.schema().field(0).data_type(),
+        plan2.schema().field(0).data_type()
+    );
+    DataFrame::new(ctx.state(), plan2).collect().await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_union_with_null_second_input() -> Result<()> 
{
+    assert_union_with_null_struct_roundtrips(
+        "SELECT st AS r FROM struct_rows UNION ALL SELECT NULL AS r FROM 
struct_rows",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_union_with_null_first_input() -> Result<()> {
+    assert_union_with_null_struct_roundtrips(
+        "SELECT NULL AS r FROM struct_rows UNION ALL SELECT st AS r FROM 
struct_rows",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_list() -> Result<()> {
+    roundtrip_with_ctx(
+        "SELECT st IN (named_struct('a', 1, 'b', 'x'), named_struct('a', 2, 
'b', 'y')) \
+         FROM struct_rows",
+        struct_rows_ctx().await?,
+    )
+    .await?;
+    Ok(())
+}
+
+// Keep these plans unoptimized before serialization so the consumer sees the
+// literal and relation shapes emitted by another Substrait producer.
+#[tokio::test]
+async fn roundtrip_struct_literal_dictionary_values() -> Result<()> {

Review Comment:
   In general, I see the approach taken for testing in this PR is pretty sloppy:
   - Import statement inside function bodies rather than at the top of the file
   - Unreasonably long and hard to read test cases
   - Inconsistent shape with existing tests previously contributed to this file
   
   I'd aim to rework the tests here, hopefully following a consistent shape 
with the ones already committed here. I'd expect every roundtrip test to be of 
a shape like:
   
   ```rust
   #[tokio::test]
   async fn roundtrip_struct_literal_union_with_null_first_input() -> 
Result<()> {
       assert_union_with_null_struct_roundtrips(
           "SELECT NULL AS r FROM struct_rows UNION ALL SELECT st AS r FROM 
struct_rows",
       )
       .await
   }
   ```
   
   Unfortunately, I see some sloppiness in the testing approach already made 
its way to `main` in https://github.com/apache/datafusion/pull/20977, which is 
unfortunate, but I'd try to not continue that pattern and deliver some readable 
tests here.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to