This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 9051efdb87 feat: switch VirtualTable producer to use expressions field
instead of deprecated values (#23672)
9051efdb87 is described below
commit 9051efdb87cd417b632e69759ed65e119bd06360
Author: eliot1480 <[email protected]>
AuthorDate: Mon Aug 3 02:31:14 2026 -0400
feat: switch VirtualTable producer to use expressions field instead of
deprecated values (#23672)
## Which issue does this PR close?
- Closes #23608
## Rationale for this change
Since the VirtualTables.values field has been deprecated and is soon to
be removed in the next substrait update, we should make sure to delete
all instances of it to prevent the presence of dead code.
This PR switches the producer to generate VirtualTables using only the
expressions field. A later PR will be released to fully remove all
instances of the `values` field. The reason removal will be separate is
because we want to give users a chance to update any stored plans, and
then remove values.
The consumer also needed an update, because since the producer only
emits expressions, a new gap was introduced where expressions never uses
named_struct.names to rename nested fields (structs/lists) the way the
values/literal path does. So for flat/scalar columns, expressions would
work fine, but for struct/list-typed columns, the consumer would
silently drop original nested field names and fall back to positional
defaults.
## What changes are included in this PR?
Producer now emits VirtualTable.expressions for Values and
EmptyRelation. Converted 17 direct virtualTable.values entries across 10
JSON test plans to virtualTable.expressions. Consumer now adds support
for struct field-name preservation when reading from only expressions.
## Are these changes tested?
Yes they are. Tests ran:
```
cargo check -p datafusion-substrait
cargo test -p datafusion-substrait -- test substrait_integration
roundtrip_values
cargo test -p datafusion-substrait -- test substrait_integration
consumer_integration
cargo test -p datafusion-substrait -- test substrait_integration
builtin_expr_semantics_tests
cargo test -p datafusion-substrait -- test substrait_integration
non_nullable_lists
```
jq validation for JSON fixtures
## Are there any user-facing changes?
No.
---
.../src/logical_plan/consumer/rel/read_rel.rs | 45 +++-
.../src/logical_plan/producer/rel/read_rel.rs | 78 +-----
.../test_plans/join_with_expression_key.json | 189 ++++++++-----
.../mixed_join_equal_and_indistinct.json | 296 ++++++++++++++++++++-
.../mixed_join_equal_and_indistinct_left.json | 296 ++++++++++++++++++++-
.../tests/testdata/test_plans/multiple_joins.json | 170 +++++++-----
.../test_plans/non_nullable_lists.substrait.json | 36 +--
.../test_plans/scalar_fn_logb_expr.substrait.json | 51 ++--
.../scalar_fn_to_between_expr.substrait.json | 69 +++--
..._to_built_in_binary_expr_and_not.substrait.json | 99 ++++---
...r_fn_to_built_in_binary_expr_xor.substrait.json | 99 ++++---
.../select_count_from_select_1.substrait.json | 8 +-
12 files changed, 1082 insertions(+), 354 deletions(-)
diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
index 832110e111..78951a3aff 100644
--- a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
@@ -148,19 +148,48 @@ pub async fn from_read_rel(
let values = if !vt.expressions.is_empty() {
let mut exprs = vec![];
for row in &vt.expressions {
+ if row.fields.len() != substrait_schema.fields().len() {
+ return substrait_err!(
+ "Field count mismatch: expected {} fields but
found {} in virtual table row",
+ substrait_schema.fields().len(),
+ row.fields.len()
+ );
+ }
+
let mut row_exprs = vec![];
+ let mut name_idx = 0;
for expression in &row.fields {
- let expr = consumer
- .consume_expression(expression, &substrait_schema)
- .await?;
+ // Top-level names are provided through schema
+ // Each expression consumes at least one name, and
Literals may consume additional names.
+ name_idx += 1;
+ let expr = match expression.rex_type.as_ref() {
+
Some(substrait::proto::expression::RexType::Literal(lit)) => {
+ // Values literals need 'named_struct.names'
so nested struct fields keep their names from the ReadRel base schema.
+ // This is important for nested struct fields
to retain their names.
+ Expr::Literal(
+ from_substrait_literal(
+ consumer,
+ lit,
+ &named_struct.names,
+ &mut name_idx,
+ )?,
+ None,
+ )
+ }
+ _ => {
+ consumer
+ .consume_expression(expression,
&substrait_schema)
+ .await?
+ }
+ };
row_exprs.push(expr);
}
- // For expressions, validate against top-level schema
fields, not nested names
- if row_exprs.len() != substrait_schema.fields().len() {
+
+ if name_idx != named_struct.names.len() {
return substrait_err!(
- "Field count mismatch: expected {} fields but
found {} in virtual table row",
- substrait_schema.fields().len(),
- row_exprs.len()
+ "Names list must match exactly to nested schema,
but found {} uses for {} names",
+ name_idx,
+ named_struct.names.len()
);
}
exprs.push(row_exprs);
diff --git a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs
b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs
index 8dfbb36d37..900273bf8e 100644
--- a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs
+++ b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs
@@ -15,55 +15,19 @@
// specific language governing permissions and limitations
// under the License.
-use crate::logical_plan::producer::{
- SubstraitProducer, to_substrait_literal, to_substrait_named_struct,
-};
+use crate::logical_plan::producer::{SubstraitProducer,
to_substrait_named_struct};
use datafusion::common::{DFSchema, ToDFSchema, substrait_datafusion_err};
use datafusion::logical_expr::utils::conjunction;
use datafusion::logical_expr::{EmptyRelation, Expr, TableScan, Values};
use datafusion::scalar::ScalarValue;
use std::sync::Arc;
use substrait::proto::expression::MaskExpression;
-use substrait::proto::expression::literal::Struct as LiteralStruct;
use substrait::proto::expression::mask_expression::{StructItem, StructSelect};
use substrait::proto::expression::nested::Struct as NestedStruct;
use substrait::proto::read_rel::{NamedTable, ReadType, VirtualTable};
use substrait::proto::rel::RelType;
use substrait::proto::{ReadRel, Rel};
-/// Converts rows of literal expressions into Substrait literal structs.
-///
-/// Each row is expected to contain only `Expr::Literal` or `Expr::Alias`
wrapping literals.
-/// Aliases are unwrapped and the underlying literal is converted.
-fn convert_literal_rows(
- producer: &mut impl SubstraitProducer,
- rows: &[Vec<Expr>],
-) -> datafusion::common::Result<Vec<LiteralStruct>> {
- rows.iter()
- .map(|row| {
- let fields = row
- .iter()
- .map(|expr| match expr {
- Expr::Literal(sv, _) => to_substrait_literal(producer, sv),
- Expr::Alias(alias) => match alias.expr.as_ref() {
- // The schema gives us the names, so we can skip
aliases
- Expr::Literal(sv, _) => to_substrait_literal(producer,
sv),
- _ => Err(substrait_datafusion_err!(
- "Only literal types can be aliased in Virtual
Tables, got: {}",
- alias.expr.variant_name()
- )),
- },
- _ => Err(substrait_datafusion_err!(
- "Only literal types and aliases are supported in
Virtual Tables, got: {}",
- expr.variant_name()
- )),
- })
- .collect::<datafusion::common::Result<_>>()?;
- Ok(LiteralStruct { fields })
- })
- .collect()
-}
-
/// Converts rows of arbitrary expressions into Substrait nested structs.
///
/// Validates that each row has the expected schema length and converts each
expression
@@ -163,6 +127,7 @@ pub fn from_empty_relation(
let base_schema = to_substrait_named_struct(producer, &e.schema)?;
let read_type = if e.produce_one_row {
+ let empty_schema = Arc::new(DFSchema::empty());
// Create one row with default scalar values for each field in the
schema.
// For example, an Int32 field gets Int32(NULL), a Utf8 field gets
Utf8(NULL), etc.
// This represents the "phantom row" that provides a context for
evaluating
@@ -173,25 +138,16 @@ pub fn from_empty_relation(
.iter()
.map(|f| {
let scalar = ScalarValue::try_from(f.data_type())?;
- to_substrait_literal(producer, &scalar)
+ producer.handle_expr(&Expr::Literal(scalar, None),
&empty_schema)
})
.collect::<datafusion::common::Result<_>>()?;
ReadType::VirtualTable(VirtualTable {
- // Use deprecated 'values' field instead of 'expressions' because
the consumer's
- // nested expression support (RexType::Nested) is not yet
implemented.
- // The 'values' field uses literal::Struct which the consumer can
properly
- // deserialize with field name preservation.
- #[expect(deprecated)]
- values: vec![LiteralStruct { fields }],
- expressions: vec![],
+ expressions: vec![NestedStruct { fields }],
+ ..Default::default()
})
} else {
- ReadType::VirtualTable(VirtualTable {
- #[expect(deprecated)]
- values: vec![],
- expressions: vec![],
- })
+ ReadType::VirtualTable(VirtualTable::default())
};
Ok(Box::new(Rel {
rel_type: Some(RelType::Read(Box::new(ReadRel {
@@ -212,23 +168,8 @@ pub fn from_values(
) -> datafusion::common::Result<Box<Rel>> {
let schema_len = v.schema.fields().len();
let empty_schema = Arc::new(DFSchema::empty());
-
- let use_literals = v.values.iter().all(|row| {
- row.iter().all(|expr| match expr {
- Expr::Literal(_, _) => true,
- Expr::Alias(alias) => matches!(alias.expr.as_ref(),
Expr::Literal(_, _)),
- _ => false,
- })
- });
-
- let (values, expressions) = if use_literals {
- let values = convert_literal_rows(producer, &v.values)?;
- (values, vec![])
- } else {
- let expressions =
- convert_expression_rows(producer, &v.values, schema_len,
&empty_schema)?;
- (vec![], expressions)
- };
+ let expressions =
+ convert_expression_rows(producer, &v.values, schema_len,
&empty_schema)?;
Ok(Box::new(Rel {
rel_type: Some(RelType::Read(Box::new(ReadRel {
common: None,
@@ -237,10 +178,9 @@ pub fn from_values(
best_effort_filter: None,
projection: None,
advanced_extension: None,
- #[expect(deprecated)]
read_type: Some(ReadType::VirtualTable(VirtualTable {
- values,
expressions,
+ ..Default::default()
})),
}))),
}))
diff --git
a/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json
b/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json
index 73fa06eea5..8a81a9a0c7 100644
---
a/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json
+++
b/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json
@@ -100,29 +100,52 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "string": "aaa",
- "nullable": true
- }, {
- "string": "host-a",
- "nullable": true
- }, {
- "i64": "128",
- "nullable": true
- }]
- }, {
- "fields": [{
- "string": "bbb",
- "nullable": true
- }, {
- "string": "host-b",
- "nullable": true
- }, {
- "i64": "256",
- "nullable": true
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "aaa",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "host-a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "128",
+ "nullable": true
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "bbb",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "host-b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "256",
+ "nullable": true
+ }
+ }
+ ]
+ }
+ ]
}
}
},
@@ -293,23 +316,40 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "string": "host-a",
- "nullable": true
- }, {
- "i64": "107",
- "nullable": true
- }]
- }, {
- "fields": [{
- "string": "host-b",
- "nullable": true
- }, {
- "i64": "214",
- "nullable": true
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "host-a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "107",
+ "nullable": true
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "host-b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "214",
+ "nullable": true
+ }
+ }
+ ]
+ }
+ ]
}
}
},
@@ -365,29 +405,52 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "string": "aaa",
- "nullable": true
- }, {
- "string": "host-a",
- "nullable": true
- }, {
- "i64": "128",
- "nullable": true
- }]
- }, {
- "fields": [{
- "string": "bbb",
- "nullable": true
- }, {
- "string": "host-b",
- "nullable": true
- }, {
- "i64": "256",
- "nullable": true
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "aaa",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "host-a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "128",
+ "nullable": true
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "bbb",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "host-b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "256",
+ "nullable": true
+ }
+ }
+ ]
+ }
+ ]
}
}
},
diff --git
a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json
b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json
index 642256c562..13c1e5899d 100644
---
a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json
+++
b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json
@@ -24,13 +24,147 @@
}
},
"virtualTable": {
- "values": [
- { "fields": [{ "string": "1", "nullable": false }, {
"string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },
- { "fields": [{ "string": "2", "nullable": false }, {
"string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },
- { "fields": [{ "string": "3", "nullable": false }, { "null":
{ "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, {
"string": "c3", "nullable": false }] },
- { "fields": [{ "string": "4", "nullable": false }, { "null":
{ "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, {
"string": "c4", "nullable": false }] },
- { "fields": [{ "string": "5", "nullable": false }, {
"string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },
- { "fields": [{ "string": "6", "nullable": false }, {
"string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "1",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c1",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "2",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c2",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "3",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c3",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "4",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c4",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "5",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "e",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c5",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "6",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "f",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c6",
+ "nullable": false
+ }
+ }
+ ]
+ }
]
}
}
@@ -50,13 +184,147 @@
}
},
"virtualTable": {
- "values": [
- { "fields": [{ "string": "1", "nullable": false }, {
"string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },
- { "fields": [{ "string": "2", "nullable": false }, {
"string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },
- { "fields": [{ "string": "3", "nullable": false }, { "null":
{ "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, {
"string": "c3", "nullable": false }] },
- { "fields": [{ "string": "4", "nullable": false }, { "null":
{ "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, {
"string": "c4", "nullable": false }] },
- { "fields": [{ "string": "5", "nullable": false }, {
"string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },
- { "fields": [{ "string": "6", "nullable": false }, {
"string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "1",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c1",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "2",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c2",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "3",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c3",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "4",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c4",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "5",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "e",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c5",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "6",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "f",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c6",
+ "nullable": false
+ }
+ }
+ ]
+ }
]
}
}
diff --git
a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json
b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json
index f16672947e..481bba44d8 100644
---
a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json
+++
b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json
@@ -24,13 +24,147 @@
}
},
"virtualTable": {
- "values": [
- { "fields": [{ "string": "1", "nullable": false }, {
"string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },
- { "fields": [{ "string": "2", "nullable": false }, {
"string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },
- { "fields": [{ "string": "3", "nullable": false }, { "null":
{ "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, {
"string": "c3", "nullable": false }] },
- { "fields": [{ "string": "4", "nullable": false }, { "null":
{ "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, {
"string": "c4", "nullable": false }] },
- { "fields": [{ "string": "5", "nullable": false }, {
"string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },
- { "fields": [{ "string": "6", "nullable": false }, {
"string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "1",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c1",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "2",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c2",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "3",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c3",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "4",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c4",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "5",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "e",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c5",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "6",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "f",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c6",
+ "nullable": false
+ }
+ }
+ ]
+ }
]
}
}
@@ -50,13 +184,147 @@
}
},
"virtualTable": {
- "values": [
- { "fields": [{ "string": "1", "nullable": false }, {
"string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },
- { "fields": [{ "string": "2", "nullable": false }, {
"string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },
- { "fields": [{ "string": "3", "nullable": false }, { "null":
{ "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, {
"string": "c3", "nullable": false }] },
- { "fields": [{ "string": "4", "nullable": false }, { "null":
{ "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, {
"string": "c4", "nullable": false }] },
- { "fields": [{ "string": "5", "nullable": false }, {
"string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },
- { "fields": [{ "string": "6", "nullable": false }, {
"string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "1",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c1",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "2",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c2",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "3",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c3",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "4",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c4",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "5",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "e",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c5",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "6",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "f",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c6",
+ "nullable": false
+ }
+ }
+ ]
+ }
]
}
}
diff --git a/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json
b/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json
index e88cce648d..15c0313b43 100644
--- a/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json
+++ b/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json
@@ -72,19 +72,30 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "i64": "1",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }, {
- "fields": [{
- "i64": "2",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "1",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "2",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ }
+ ]
}
}
},
@@ -153,27 +164,44 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "i64": "1",
- "nullable": true,
- "typeVariationReference": 0
- }, {
- "string": "info",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }, {
- "fields": [{
- "i64": "2",
- "nullable": true,
- "typeVariationReference": 0
- }, {
- "string": "low",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "1",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ },
+ {
+ "literal": {
+ "string": "info",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "2",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ },
+ {
+ "literal": {
+ "string": "low",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ }
+ ]
}
}
},
@@ -272,19 +300,30 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "i64": "1",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }, {
- "fields": [{
- "i64": "2",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "1",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "2",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ }
+ ]
}
}
},
@@ -389,19 +428,30 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "i64": "1",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }, {
- "fields": [{
- "i64": "2",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "1",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "2",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ }
+ ]
}
}
},
diff --git
a/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json
b/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json
index e1c5574f8b..e29c000ee6 100644
---
a/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json
+++
b/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json
@@ -34,26 +34,28 @@
}
},
"virtualTable": {
- "values": [
+ "expressions": [
{
"fields": [
{
- "list": {
- "values": [
- {
- "i32": 1,
- "nullable": false,
- "typeVariationReference": 0
- },
- {
- "i32": 2,
- "nullable": false,
- "typeVariationReference": 0
- }
- ]
- },
- "nullable": false,
- "typeVariationReference": 0
+ "literal": {
+ "list": {
+ "values": [
+ {
+ "i32": 1,
+ "nullable": false,
+ "typeVariationReference": 0
+ },
+ {
+ "i32": 2,
+ "nullable": false,
+ "typeVariationReference": 0
+ }
+ ]
+ },
+ "nullable": false,
+ "typeVariationReference": 0
+ }
}
]
}
diff --git
a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json
b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json
index eeaf5a3dd8..d5209a683d 100644
---
a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json
+++
b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json
@@ -85,23 +85,40 @@
"direct": {}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "fp32": 1.0,
- "nullable": false
- }, {
- "fp32": 10.0,
- "nullable": false
- }]
- }, {
- "fields": [{
- "fp32": 100.0,
- "nullable": false
- }, {
- "fp32": 10.0,
- "nullable": false
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "fp32": 1.0,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "fp32": 10.0,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "fp32": 100.0,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "fp32": 10.0,
+ "nullable": false
+ }
+ }
+ ]
+ }
+ ]
}
}
}
diff --git
a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json
b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json
index 6749a301b1..f609d26138 100644
---
a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json
+++
b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json
@@ -106,29 +106,52 @@
"direct": {}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "i8": 2,
- "nullable": false
- }, {
- "i8": 1,
- "nullable": false
- }, {
- "i8": 3,
- "nullable": false
- }]
- }, {
- "fields": [{
- "i8": 4,
- "nullable": false
- }, {
- "i8": 1,
- "nullable": false
- }, {
- "i8": 2,
- "nullable": false
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "i8": 2,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "i8": 1,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "i8": 3,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "i8": 4,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "i8": 1,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "i8": 2,
+ "nullable": false
+ }
+ }
+ ]
+ }
+ ]
}
}
}
diff --git
a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json
b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json
index 8365b1edfe..5d91342257 100644
---
a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json
+++
b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json
@@ -85,39 +85,72 @@
"direct": {}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "boolean": true,
- "nullable": false
- }, {
- "boolean": true,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": true,
- "nullable": false
- }, {
- "boolean": false,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": false,
- "nullable": false
- }, {
- "boolean": true,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": false,
- "nullable": false
- }, {
- "boolean": false,
- "nullable": false
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ }
+ ]
+ }
+ ]
}
}
}
diff --git
a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json
b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json
index cfd760de89..2514c0afc9 100644
---
a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json
+++
b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json
@@ -85,39 +85,72 @@
"direct": {}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "boolean": true,
- "nullable": false
- }, {
- "boolean": true,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": true,
- "nullable": false
- }, {
- "boolean": false,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": false,
- "nullable": false
- }, {
- "boolean": true,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": false,
- "nullable": false
- }, {
- "boolean": false,
- "nullable": false
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ }
+ ]
+ }
+ ]
}
}
}
diff --git
a/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json
b/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json
index e9f6795880..b0d4ba4813 100644
---
a/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json
+++
b/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json
@@ -43,12 +43,14 @@
}
},
"virtualTable": {
- "values": [
+ "expressions": [
{
"fields": [
{
- "i64": "0",
- "nullable": false
+ "literal": {
+ "i64": "0",
+ "nullable": false
+ }
}
]
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
