This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23672-37baf4293d82f05b868ddde40017f3a2fd41511c in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 9051efdb87cd417b632e69759ed65e119bd06360 Author: eliot1480 <[email protected]> AuthorDate: Mon Aug 3 02:31:14 2026 -0400 feat: switch VirtualTable producer to use expressions field instead of deprecated values (#23672) ## Which issue does this PR close? - Closes #23608 ## Rationale for this change Since the VirtualTables.values field has been deprecated and is soon to be removed in the next substrait update, we should make sure to delete all instances of it to prevent the presence of dead code. This PR switches the producer to generate VirtualTables using only the expressions field. A later PR will be released to fully remove all instances of the `values` field. The reason removal will be separate is because we want to give users a chance to update any stored plans, and then remove values. The consumer also needed an update, because since the producer only emits expressions, a new gap was introduced where expressions never uses named_struct.names to rename nested fields (structs/lists) the way the values/literal path does. So for flat/scalar columns, expressions would work fine, but for struct/list-typed columns, the consumer would silently drop original nested field names and fall back to positional defaults. ## What changes are included in this PR? Producer now emits VirtualTable.expressions for Values and EmptyRelation. Converted 17 direct virtualTable.values entries across 10 JSON test plans to virtualTable.expressions. Consumer now adds support for struct field-name preservation when reading from only expressions. ## Are these changes tested? Yes they are. Tests ran: ``` cargo check -p datafusion-substrait cargo test -p datafusion-substrait -- test substrait_integration roundtrip_values cargo test -p datafusion-substrait -- test substrait_integration consumer_integration cargo test -p datafusion-substrait -- test substrait_integration builtin_expr_semantics_tests cargo test -p datafusion-substrait -- test substrait_integration non_nullable_lists ``` jq validation for JSON fixtures ## Are there any user-facing changes? No. --- .../src/logical_plan/consumer/rel/read_rel.rs | 45 +++- .../src/logical_plan/producer/rel/read_rel.rs | 78 +----- .../test_plans/join_with_expression_key.json | 189 ++++++++----- .../mixed_join_equal_and_indistinct.json | 296 ++++++++++++++++++++- .../mixed_join_equal_and_indistinct_left.json | 296 ++++++++++++++++++++- .../tests/testdata/test_plans/multiple_joins.json | 170 +++++++----- .../test_plans/non_nullable_lists.substrait.json | 36 +-- .../test_plans/scalar_fn_logb_expr.substrait.json | 51 ++-- .../scalar_fn_to_between_expr.substrait.json | 69 +++-- ..._to_built_in_binary_expr_and_not.substrait.json | 99 ++++--- ...r_fn_to_built_in_binary_expr_xor.substrait.json | 99 ++++--- .../select_count_from_select_1.substrait.json | 8 +- 12 files changed, 1082 insertions(+), 354 deletions(-) diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs index 832110e111..78951a3aff 100644 --- a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs +++ b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs @@ -148,19 +148,48 @@ pub async fn from_read_rel( let values = if !vt.expressions.is_empty() { let mut exprs = vec![]; for row in &vt.expressions { + if row.fields.len() != substrait_schema.fields().len() { + return substrait_err!( + "Field count mismatch: expected {} fields but found {} in virtual table row", + substrait_schema.fields().len(), + row.fields.len() + ); + } + let mut row_exprs = vec![]; + let mut name_idx = 0; for expression in &row.fields { - let expr = consumer - .consume_expression(expression, &substrait_schema) - .await?; + // Top-level names are provided through schema + // Each expression consumes at least one name, and Literals may consume additional names. + name_idx += 1; + let expr = match expression.rex_type.as_ref() { + Some(substrait::proto::expression::RexType::Literal(lit)) => { + // Values literals need 'named_struct.names' so nested struct fields keep their names from the ReadRel base schema. + // This is important for nested struct fields to retain their names. + Expr::Literal( + from_substrait_literal( + consumer, + lit, + &named_struct.names, + &mut name_idx, + )?, + None, + ) + } + _ => { + consumer + .consume_expression(expression, &substrait_schema) + .await? + } + }; row_exprs.push(expr); } - // For expressions, validate against top-level schema fields, not nested names - if row_exprs.len() != substrait_schema.fields().len() { + + if name_idx != named_struct.names.len() { return substrait_err!( - "Field count mismatch: expected {} fields but found {} in virtual table row", - substrait_schema.fields().len(), - row_exprs.len() + "Names list must match exactly to nested schema, but found {} uses for {} names", + name_idx, + named_struct.names.len() ); } exprs.push(row_exprs); diff --git a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs index 8dfbb36d37..900273bf8e 100644 --- a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs +++ b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs @@ -15,55 +15,19 @@ // specific language governing permissions and limitations // under the License. -use crate::logical_plan::producer::{ - SubstraitProducer, to_substrait_literal, to_substrait_named_struct, -}; +use crate::logical_plan::producer::{SubstraitProducer, to_substrait_named_struct}; use datafusion::common::{DFSchema, ToDFSchema, substrait_datafusion_err}; use datafusion::logical_expr::utils::conjunction; use datafusion::logical_expr::{EmptyRelation, Expr, TableScan, Values}; use datafusion::scalar::ScalarValue; use std::sync::Arc; use substrait::proto::expression::MaskExpression; -use substrait::proto::expression::literal::Struct as LiteralStruct; use substrait::proto::expression::mask_expression::{StructItem, StructSelect}; use substrait::proto::expression::nested::Struct as NestedStruct; use substrait::proto::read_rel::{NamedTable, ReadType, VirtualTable}; use substrait::proto::rel::RelType; use substrait::proto::{ReadRel, Rel}; -/// Converts rows of literal expressions into Substrait literal structs. -/// -/// Each row is expected to contain only `Expr::Literal` or `Expr::Alias` wrapping literals. -/// Aliases are unwrapped and the underlying literal is converted. -fn convert_literal_rows( - producer: &mut impl SubstraitProducer, - rows: &[Vec<Expr>], -) -> datafusion::common::Result<Vec<LiteralStruct>> { - rows.iter() - .map(|row| { - let fields = row - .iter() - .map(|expr| match expr { - Expr::Literal(sv, _) => to_substrait_literal(producer, sv), - Expr::Alias(alias) => match alias.expr.as_ref() { - // The schema gives us the names, so we can skip aliases - Expr::Literal(sv, _) => to_substrait_literal(producer, sv), - _ => Err(substrait_datafusion_err!( - "Only literal types can be aliased in Virtual Tables, got: {}", - alias.expr.variant_name() - )), - }, - _ => Err(substrait_datafusion_err!( - "Only literal types and aliases are supported in Virtual Tables, got: {}", - expr.variant_name() - )), - }) - .collect::<datafusion::common::Result<_>>()?; - Ok(LiteralStruct { fields }) - }) - .collect() -} - /// Converts rows of arbitrary expressions into Substrait nested structs. /// /// Validates that each row has the expected schema length and converts each expression @@ -163,6 +127,7 @@ pub fn from_empty_relation( let base_schema = to_substrait_named_struct(producer, &e.schema)?; let read_type = if e.produce_one_row { + let empty_schema = Arc::new(DFSchema::empty()); // Create one row with default scalar values for each field in the schema. // For example, an Int32 field gets Int32(NULL), a Utf8 field gets Utf8(NULL), etc. // This represents the "phantom row" that provides a context for evaluating @@ -173,25 +138,16 @@ pub fn from_empty_relation( .iter() .map(|f| { let scalar = ScalarValue::try_from(f.data_type())?; - to_substrait_literal(producer, &scalar) + producer.handle_expr(&Expr::Literal(scalar, None), &empty_schema) }) .collect::<datafusion::common::Result<_>>()?; ReadType::VirtualTable(VirtualTable { - // Use deprecated 'values' field instead of 'expressions' because the consumer's - // nested expression support (RexType::Nested) is not yet implemented. - // The 'values' field uses literal::Struct which the consumer can properly - // deserialize with field name preservation. - #[expect(deprecated)] - values: vec![LiteralStruct { fields }], - expressions: vec![], + expressions: vec![NestedStruct { fields }], + ..Default::default() }) } else { - ReadType::VirtualTable(VirtualTable { - #[expect(deprecated)] - values: vec![], - expressions: vec![], - }) + ReadType::VirtualTable(VirtualTable::default()) }; Ok(Box::new(Rel { rel_type: Some(RelType::Read(Box::new(ReadRel { @@ -212,23 +168,8 @@ pub fn from_values( ) -> datafusion::common::Result<Box<Rel>> { let schema_len = v.schema.fields().len(); let empty_schema = Arc::new(DFSchema::empty()); - - let use_literals = v.values.iter().all(|row| { - row.iter().all(|expr| match expr { - Expr::Literal(_, _) => true, - Expr::Alias(alias) => matches!(alias.expr.as_ref(), Expr::Literal(_, _)), - _ => false, - }) - }); - - let (values, expressions) = if use_literals { - let values = convert_literal_rows(producer, &v.values)?; - (values, vec![]) - } else { - let expressions = - convert_expression_rows(producer, &v.values, schema_len, &empty_schema)?; - (vec![], expressions) - }; + let expressions = + convert_expression_rows(producer, &v.values, schema_len, &empty_schema)?; Ok(Box::new(Rel { rel_type: Some(RelType::Read(Box::new(ReadRel { common: None, @@ -237,10 +178,9 @@ pub fn from_values( best_effort_filter: None, projection: None, advanced_extension: None, - #[expect(deprecated)] read_type: Some(ReadType::VirtualTable(VirtualTable { - values, expressions, + ..Default::default() })), }))), })) diff --git a/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json b/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json index 73fa06eea5..8a81a9a0c7 100644 --- a/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json +++ b/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json @@ -100,29 +100,52 @@ } }, "virtualTable": { - "values": [{ - "fields": [{ - "string": "aaa", - "nullable": true - }, { - "string": "host-a", - "nullable": true - }, { - "i64": "128", - "nullable": true - }] - }, { - "fields": [{ - "string": "bbb", - "nullable": true - }, { - "string": "host-b", - "nullable": true - }, { - "i64": "256", - "nullable": true - }] - }] + "expressions": [ + { + "fields": [ + { + "literal": { + "string": "aaa", + "nullable": true + } + }, + { + "literal": { + "string": "host-a", + "nullable": true + } + }, + { + "literal": { + "i64": "128", + "nullable": true + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "bbb", + "nullable": true + } + }, + { + "literal": { + "string": "host-b", + "nullable": true + } + }, + { + "literal": { + "i64": "256", + "nullable": true + } + } + ] + } + ] } } }, @@ -293,23 +316,40 @@ } }, "virtualTable": { - "values": [{ - "fields": [{ - "string": "host-a", - "nullable": true - }, { - "i64": "107", - "nullable": true - }] - }, { - "fields": [{ - "string": "host-b", - "nullable": true - }, { - "i64": "214", - "nullable": true - }] - }] + "expressions": [ + { + "fields": [ + { + "literal": { + "string": "host-a", + "nullable": true + } + }, + { + "literal": { + "i64": "107", + "nullable": true + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "host-b", + "nullable": true + } + }, + { + "literal": { + "i64": "214", + "nullable": true + } + } + ] + } + ] } } }, @@ -365,29 +405,52 @@ } }, "virtualTable": { - "values": [{ - "fields": [{ - "string": "aaa", - "nullable": true - }, { - "string": "host-a", - "nullable": true - }, { - "i64": "128", - "nullable": true - }] - }, { - "fields": [{ - "string": "bbb", - "nullable": true - }, { - "string": "host-b", - "nullable": true - }, { - "i64": "256", - "nullable": true - }] - }] + "expressions": [ + { + "fields": [ + { + "literal": { + "string": "aaa", + "nullable": true + } + }, + { + "literal": { + "string": "host-a", + "nullable": true + } + }, + { + "literal": { + "i64": "128", + "nullable": true + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "bbb", + "nullable": true + } + }, + { + "literal": { + "string": "host-b", + "nullable": true + } + }, + { + "literal": { + "i64": "256", + "nullable": true + } + } + ] + } + ] } } }, diff --git a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json index 642256c562..13c1e5899d 100644 --- a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json +++ b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json @@ -24,13 +24,147 @@ } }, "virtualTable": { - "values": [ - { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] }, - { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] }, - { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] }, - { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] }, - { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] }, - { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] } + "expressions": [ + { + "fields": [ + { + "literal": { + "string": "1", + "nullable": false + } + }, + { + "literal": { + "string": "a", + "nullable": true + } + }, + { + "literal": { + "string": "c1", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "2", + "nullable": false + } + }, + { + "literal": { + "string": "b", + "nullable": true + } + }, + { + "literal": { + "string": "c2", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "3", + "nullable": false + } + }, + { + "literal": { + "null": { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "nullable": true + } + }, + { + "literal": { + "string": "c3", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "4", + "nullable": false + } + }, + { + "literal": { + "null": { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "nullable": true + } + }, + { + "literal": { + "string": "c4", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "5", + "nullable": false + } + }, + { + "literal": { + "string": "e", + "nullable": true + } + }, + { + "literal": { + "string": "c5", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "6", + "nullable": false + } + }, + { + "literal": { + "string": "f", + "nullable": true + } + }, + { + "literal": { + "string": "c6", + "nullable": false + } + } + ] + } ] } } @@ -50,13 +184,147 @@ } }, "virtualTable": { - "values": [ - { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] }, - { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] }, - { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] }, - { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] }, - { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] }, - { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] } + "expressions": [ + { + "fields": [ + { + "literal": { + "string": "1", + "nullable": false + } + }, + { + "literal": { + "string": "a", + "nullable": true + } + }, + { + "literal": { + "string": "c1", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "2", + "nullable": false + } + }, + { + "literal": { + "string": "b", + "nullable": true + } + }, + { + "literal": { + "string": "c2", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "3", + "nullable": false + } + }, + { + "literal": { + "null": { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "nullable": true + } + }, + { + "literal": { + "string": "c3", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "4", + "nullable": false + } + }, + { + "literal": { + "null": { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "nullable": true + } + }, + { + "literal": { + "string": "c4", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "5", + "nullable": false + } + }, + { + "literal": { + "string": "e", + "nullable": true + } + }, + { + "literal": { + "string": "c5", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "6", + "nullable": false + } + }, + { + "literal": { + "string": "f", + "nullable": true + } + }, + { + "literal": { + "string": "c6", + "nullable": false + } + } + ] + } ] } } diff --git a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json index f16672947e..481bba44d8 100644 --- a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json +++ b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json @@ -24,13 +24,147 @@ } }, "virtualTable": { - "values": [ - { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] }, - { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] }, - { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] }, - { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] }, - { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] }, - { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] } + "expressions": [ + { + "fields": [ + { + "literal": { + "string": "1", + "nullable": false + } + }, + { + "literal": { + "string": "a", + "nullable": true + } + }, + { + "literal": { + "string": "c1", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "2", + "nullable": false + } + }, + { + "literal": { + "string": "b", + "nullable": true + } + }, + { + "literal": { + "string": "c2", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "3", + "nullable": false + } + }, + { + "literal": { + "null": { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "nullable": true + } + }, + { + "literal": { + "string": "c3", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "4", + "nullable": false + } + }, + { + "literal": { + "null": { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "nullable": true + } + }, + { + "literal": { + "string": "c4", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "5", + "nullable": false + } + }, + { + "literal": { + "string": "e", + "nullable": true + } + }, + { + "literal": { + "string": "c5", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "6", + "nullable": false + } + }, + { + "literal": { + "string": "f", + "nullable": true + } + }, + { + "literal": { + "string": "c6", + "nullable": false + } + } + ] + } ] } } @@ -50,13 +184,147 @@ } }, "virtualTable": { - "values": [ - { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] }, - { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] }, - { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] }, - { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] }, - { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] }, - { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] } + "expressions": [ + { + "fields": [ + { + "literal": { + "string": "1", + "nullable": false + } + }, + { + "literal": { + "string": "a", + "nullable": true + } + }, + { + "literal": { + "string": "c1", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "2", + "nullable": false + } + }, + { + "literal": { + "string": "b", + "nullable": true + } + }, + { + "literal": { + "string": "c2", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "3", + "nullable": false + } + }, + { + "literal": { + "null": { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "nullable": true + } + }, + { + "literal": { + "string": "c3", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "4", + "nullable": false + } + }, + { + "literal": { + "null": { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "nullable": true + } + }, + { + "literal": { + "string": "c4", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "5", + "nullable": false + } + }, + { + "literal": { + "string": "e", + "nullable": true + } + }, + { + "literal": { + "string": "c5", + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "string": "6", + "nullable": false + } + }, + { + "literal": { + "string": "f", + "nullable": true + } + }, + { + "literal": { + "string": "c6", + "nullable": false + } + } + ] + } ] } } diff --git a/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json b/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json index e88cce648d..15c0313b43 100644 --- a/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json +++ b/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json @@ -72,19 +72,30 @@ } }, "virtualTable": { - "values": [{ - "fields": [{ - "i64": "1", - "nullable": true, - "typeVariationReference": 0 - }] - }, { - "fields": [{ - "i64": "2", - "nullable": true, - "typeVariationReference": 0 - }] - }] + "expressions": [ + { + "fields": [ + { + "literal": { + "i64": "1", + "nullable": true, + "typeVariationReference": 0 + } + } + ] + }, + { + "fields": [ + { + "literal": { + "i64": "2", + "nullable": true, + "typeVariationReference": 0 + } + } + ] + } + ] } } }, @@ -153,27 +164,44 @@ } }, "virtualTable": { - "values": [{ - "fields": [{ - "i64": "1", - "nullable": true, - "typeVariationReference": 0 - }, { - "string": "info", - "nullable": true, - "typeVariationReference": 0 - }] - }, { - "fields": [{ - "i64": "2", - "nullable": true, - "typeVariationReference": 0 - }, { - "string": "low", - "nullable": true, - "typeVariationReference": 0 - }] - }] + "expressions": [ + { + "fields": [ + { + "literal": { + "i64": "1", + "nullable": true, + "typeVariationReference": 0 + } + }, + { + "literal": { + "string": "info", + "nullable": true, + "typeVariationReference": 0 + } + } + ] + }, + { + "fields": [ + { + "literal": { + "i64": "2", + "nullable": true, + "typeVariationReference": 0 + } + }, + { + "literal": { + "string": "low", + "nullable": true, + "typeVariationReference": 0 + } + } + ] + } + ] } } }, @@ -272,19 +300,30 @@ } }, "virtualTable": { - "values": [{ - "fields": [{ - "i64": "1", - "nullable": true, - "typeVariationReference": 0 - }] - }, { - "fields": [{ - "i64": "2", - "nullable": true, - "typeVariationReference": 0 - }] - }] + "expressions": [ + { + "fields": [ + { + "literal": { + "i64": "1", + "nullable": true, + "typeVariationReference": 0 + } + } + ] + }, + { + "fields": [ + { + "literal": { + "i64": "2", + "nullable": true, + "typeVariationReference": 0 + } + } + ] + } + ] } } }, @@ -389,19 +428,30 @@ } }, "virtualTable": { - "values": [{ - "fields": [{ - "i64": "1", - "nullable": true, - "typeVariationReference": 0 - }] - }, { - "fields": [{ - "i64": "2", - "nullable": true, - "typeVariationReference": 0 - }] - }] + "expressions": [ + { + "fields": [ + { + "literal": { + "i64": "1", + "nullable": true, + "typeVariationReference": 0 + } + } + ] + }, + { + "fields": [ + { + "literal": { + "i64": "2", + "nullable": true, + "typeVariationReference": 0 + } + } + ] + } + ] } } }, diff --git a/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json b/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json index e1c5574f8b..e29c000ee6 100644 --- a/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json +++ b/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json @@ -34,26 +34,28 @@ } }, "virtualTable": { - "values": [ + "expressions": [ { "fields": [ { - "list": { - "values": [ - { - "i32": 1, - "nullable": false, - "typeVariationReference": 0 - }, - { - "i32": 2, - "nullable": false, - "typeVariationReference": 0 - } - ] - }, - "nullable": false, - "typeVariationReference": 0 + "literal": { + "list": { + "values": [ + { + "i32": 1, + "nullable": false, + "typeVariationReference": 0 + }, + { + "i32": 2, + "nullable": false, + "typeVariationReference": 0 + } + ] + }, + "nullable": false, + "typeVariationReference": 0 + } } ] } diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json index eeaf5a3dd8..d5209a683d 100644 --- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json +++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json @@ -85,23 +85,40 @@ "direct": {} }, "virtualTable": { - "values": [{ - "fields": [{ - "fp32": 1.0, - "nullable": false - }, { - "fp32": 10.0, - "nullable": false - }] - }, { - "fields": [{ - "fp32": 100.0, - "nullable": false - }, { - "fp32": 10.0, - "nullable": false - }] - }] + "expressions": [ + { + "fields": [ + { + "literal": { + "fp32": 1.0, + "nullable": false + } + }, + { + "literal": { + "fp32": 10.0, + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "fp32": 100.0, + "nullable": false + } + }, + { + "literal": { + "fp32": 10.0, + "nullable": false + } + } + ] + } + ] } } } diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json index 6749a301b1..f609d26138 100644 --- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json +++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json @@ -106,29 +106,52 @@ "direct": {} }, "virtualTable": { - "values": [{ - "fields": [{ - "i8": 2, - "nullable": false - }, { - "i8": 1, - "nullable": false - }, { - "i8": 3, - "nullable": false - }] - }, { - "fields": [{ - "i8": 4, - "nullable": false - }, { - "i8": 1, - "nullable": false - }, { - "i8": 2, - "nullable": false - }] - }] + "expressions": [ + { + "fields": [ + { + "literal": { + "i8": 2, + "nullable": false + } + }, + { + "literal": { + "i8": 1, + "nullable": false + } + }, + { + "literal": { + "i8": 3, + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "i8": 4, + "nullable": false + } + }, + { + "literal": { + "i8": 1, + "nullable": false + } + }, + { + "literal": { + "i8": 2, + "nullable": false + } + } + ] + } + ] } } } diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json index 8365b1edfe..5d91342257 100644 --- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json +++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json @@ -85,39 +85,72 @@ "direct": {} }, "virtualTable": { - "values": [{ - "fields": [{ - "boolean": true, - "nullable": false - }, { - "boolean": true, - "nullable": false - }] - }, { - "fields": [{ - "boolean": true, - "nullable": false - }, { - "boolean": false, - "nullable": false - }] - }, { - "fields": [{ - "boolean": false, - "nullable": false - }, { - "boolean": true, - "nullable": false - }] - }, { - "fields": [{ - "boolean": false, - "nullable": false - }, { - "boolean": false, - "nullable": false - }] - }] + "expressions": [ + { + "fields": [ + { + "literal": { + "boolean": true, + "nullable": false + } + }, + { + "literal": { + "boolean": true, + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "boolean": true, + "nullable": false + } + }, + { + "literal": { + "boolean": false, + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "boolean": false, + "nullable": false + } + }, + { + "literal": { + "boolean": true, + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "boolean": false, + "nullable": false + } + }, + { + "literal": { + "boolean": false, + "nullable": false + } + } + ] + } + ] } } } diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json index cfd760de89..2514c0afc9 100644 --- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json +++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json @@ -85,39 +85,72 @@ "direct": {} }, "virtualTable": { - "values": [{ - "fields": [{ - "boolean": true, - "nullable": false - }, { - "boolean": true, - "nullable": false - }] - }, { - "fields": [{ - "boolean": true, - "nullable": false - }, { - "boolean": false, - "nullable": false - }] - }, { - "fields": [{ - "boolean": false, - "nullable": false - }, { - "boolean": true, - "nullable": false - }] - }, { - "fields": [{ - "boolean": false, - "nullable": false - }, { - "boolean": false, - "nullable": false - }] - }] + "expressions": [ + { + "fields": [ + { + "literal": { + "boolean": true, + "nullable": false + } + }, + { + "literal": { + "boolean": true, + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "boolean": true, + "nullable": false + } + }, + { + "literal": { + "boolean": false, + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "boolean": false, + "nullable": false + } + }, + { + "literal": { + "boolean": true, + "nullable": false + } + } + ] + }, + { + "fields": [ + { + "literal": { + "boolean": false, + "nullable": false + } + }, + { + "literal": { + "boolean": false, + "nullable": false + } + } + ] + } + ] } } } diff --git a/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json b/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json index e9f6795880..b0d4ba4813 100644 --- a/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json +++ b/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json @@ -43,12 +43,14 @@ } }, "virtualTable": { - "values": [ + "expressions": [ { "fields": [ { - "i64": "0", - "nullable": false + "literal": { + "i64": "0", + "nullable": false + } } ] } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
