This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 675b96c2b4 Ensure Substrait consumer can handle expressions in
VirtualTable (#16857)
675b96c2b4 is described below
commit 675b96c2b405dba7fe9c171854c60a64797becf6
Author: lorenarosati <lorena.ros...@datadoghq.com>
AuthorDate: Fri Jul 25 15:27:36 2025 -0400
Ensure Substrait consumer can handle expressions in VirtualTable (#16857)
* Handle expression and value elements in Substrait VirtualTable
* Added test
* Modified test plan, changed conditional check for clarity
* Consume expressions with empty input schema
---
.../src/logical_plan/consumer/rel/read_rel.rs | 30 ++++++++-
.../substrait/tests/cases/consumer_integration.rs | 15 +++++
.../virtual_table_with_expressions.substrait.json | 75 ++++++++++++++++++++++
3 files changed, 117 insertions(+), 3 deletions(-)
diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
index f1cbd16d2d..3ea318b214 100644
--- a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
@@ -114,14 +114,37 @@ pub async fn from_read_rel(
.await
}
Some(ReadType::VirtualTable(vt)) => {
- if vt.values.is_empty() {
+ if vt.values.is_empty() && vt.expressions.is_empty() {
return Ok(LogicalPlan::EmptyRelation(EmptyRelation {
produce_one_row: false,
schema: DFSchemaRef::new(substrait_schema),
}));
}
- let values = vt
+ let values = if !vt.expressions.is_empty() {
+ let mut exprs = vec![];
+ for row in &vt.expressions {
+ let mut name_idx = 0;
+ let mut row_exprs = vec![];
+ for expression in &row.fields {
+ name_idx += 1;
+ let expr = consumer
+ .consume_expression(expression, &DFSchema::empty())
+ .await?;
+ row_exprs.push(expr);
+ }
+ if name_idx != named_struct.names.len() {
+ return substrait_err!(
+ "Names list must match exactly to nested
schema, but found {} uses for {} names",
+ name_idx,
+ named_struct.names.len()
+ );
+ }
+ exprs.push(row_exprs);
+ }
+ exprs
+ } else {
+ vt
.values
.iter()
.map(|row| {
@@ -148,7 +171,8 @@ pub async fn from_read_rel(
}
Ok(lits)
})
- .collect::<datafusion::common::Result<_>>()?;
+ .collect::<datafusion::common::Result<_>>()?
+ };
Ok(LogicalPlan::Values(Values {
schema: DFSchemaRef::new(substrait_schema),
diff --git a/datafusion/substrait/tests/cases/consumer_integration.rs
b/datafusion/substrait/tests/cases/consumer_integration.rs
index 48bba45655..4d82f0fbd0 100644
--- a/datafusion/substrait/tests/cases/consumer_integration.rs
+++ b/datafusion/substrait/tests/cases/consumer_integration.rs
@@ -519,6 +519,21 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn test_expressions_in_virtual_table() -> Result<()> {
+ let plan_str =
+
test_plan_to_string("virtual_table_with_expressions.substrait.json").await?;
+
+ assert_snapshot!(
+ plan_str,
+ @r#"
+ Projection: dummy1 AS result1, dummy2 AS result2
+ Values: (Int64(0), Utf8("temp")), (Int64(1), Utf8("test"))
+ "#
+ );
+ Ok(())
+ }
+
#[tokio::test]
async fn test_multiple_joins() -> Result<()> {
let plan_str = test_plan_to_string("multiple_joins.json").await?;
diff --git
a/datafusion/substrait/tests/testdata/test_plans/virtual_table_with_expressions.substrait.json
b/datafusion/substrait/tests/testdata/test_plans/virtual_table_with_expressions.substrait.json
new file mode 100644
index 0000000000..2c634fa957
--- /dev/null
+++
b/datafusion/substrait/tests/testdata/test_plans/virtual_table_with_expressions.substrait.json
@@ -0,0 +1,75 @@
+{
+ "relations": [
+ {
+ "root": {
+ "input": {
+ "read": {
+ "common": {
+ "direct": {
+ }
+ },
+ "baseSchema": {
+ "names": [
+ "dummy1", "dummy2"
+ ],
+ "struct": {
+ "types": [
+ {
+ "i64": {
+ "nullability": "NULLABILITY_REQUIRED"
+ }
+ },
+ {
+ "string": {
+ "nullability": "NULLABILITY_REQUIRED"
+ }
+ }
+ ],
+ "nullability": "NULLABILITY_REQUIRED"
+ }
+ },
+ "virtualTable": {
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "0",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "temp",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "1",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "test",
+ "nullable": false
+ }
+ }
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "names": [
+ "result1", "result2"
+ ]
+ }
+ }
+ ]
+ }
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@datafusion.apache.org
For additional commands, e-mail: commits-h...@datafusion.apache.org
