This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new f39b467d10 implement nested identifier access (#12614)
f39b467d10 is described below
commit f39b467d10efa9fd356207ed707b5d3fd3196e82
Author: Lordworms <[email protected]>
AuthorDate: Fri Sep 27 16:17:03 2024 -0700
implement nested identifier access (#12614)
* adding struct_extract function
* changing logics
* Revert "adding struct_extract function"
This reverts commit 00a12bc03ea924dec14f48f29b6b4f77bc735e67.
* fix clippy
* optimize
---
datafusion/functions/src/core/planner.rs | 27 ++++----
datafusion/sql/src/expr/identifier.rs | 15 +----
datafusion/sqllogictest/test_files/struct.slt | 91 +++++++++++++++++++++++++++
3 files changed, 108 insertions(+), 25 deletions(-)
diff --git a/datafusion/functions/src/core/planner.rs
b/datafusion/functions/src/core/planner.rs
index 889f191d59..12fcb4a263 100644
--- a/datafusion/functions/src/core/planner.rs
+++ b/datafusion/functions/src/core/planner.rs
@@ -17,7 +17,7 @@
use arrow::datatypes::Field;
use datafusion_common::Result;
-use datafusion_common::{not_impl_err, Column, DFSchema, ScalarValue,
TableReference};
+use datafusion_common::{Column, DFSchema, ScalarValue, TableReference};
use datafusion_expr::expr::ScalarFunction;
use datafusion_expr::planner::{ExprPlanner, PlannerResult, RawDictionaryExpr};
use datafusion_expr::{lit, Expr};
@@ -70,19 +70,20 @@ impl ExprPlanner for CoreFunctionPlanner {
qualifier: Option<&TableReference>,
nested_names: &[String],
) -> Result<PlannerResult<Vec<Expr>>> {
- // TODO: remove when can support multiple nested identifiers
- if nested_names.len() > 1 {
- return not_impl_err!(
- "Nested identifiers not yet supported for column {}",
- Column::from((qualifier, field)).quoted_flat_name()
- );
+ let col = Expr::Column(Column::from((qualifier, field)));
+
+ // Start with the base column expression
+ let mut expr = col;
+
+ // Iterate over nested_names and create nested get_field expressions
+ for nested_name in nested_names {
+ let get_field_args = vec![expr,
lit(ScalarValue::from(nested_name.clone()))];
+ expr = Expr::ScalarFunction(ScalarFunction::new_udf(
+ crate::core::get_field(),
+ get_field_args,
+ ));
}
- let nested_name = nested_names[0].to_string();
- let col = Expr::Column(Column::from((qualifier, field)));
- let get_field_args = vec![col, lit(ScalarValue::from(nested_name))];
- Ok(PlannerResult::Planned(Expr::ScalarFunction(
- ScalarFunction::new_udf(crate::core::get_field(), get_field_args),
- )))
+ Ok(PlannerResult::Planned(expr))
}
}
diff --git a/datafusion/sql/src/expr/identifier.rs
b/datafusion/sql/src/expr/identifier.rs
index 36776c6902..b2c8312709 100644
--- a/datafusion/sql/src/expr/identifier.rs
+++ b/datafusion/sql/src/expr/identifier.rs
@@ -19,8 +19,8 @@ use arrow_schema::Field;
use sqlparser::ast::{Expr as SQLExpr, Ident};
use datafusion_common::{
- internal_err, not_impl_err, plan_datafusion_err, Column, DFSchema,
DataFusionError,
- Result, TableReference,
+ internal_err, not_impl_err, plan_datafusion_err, plan_err, Column,
DFSchema,
+ DataFusionError, Result, TableReference,
};
use datafusion_expr::planner::PlannerResult;
use datafusion_expr::{Case, Expr};
@@ -113,13 +113,6 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
.map(|id| self.ident_normalizer.normalize(id))
.collect::<Vec<_>>();
- // Currently not supporting more than one nested level
- // Though ideally once that support is in place, this code should
work with it
- // TODO: remove when can support multiple nested identifiers
- if ids.len() > 5 {
- return not_impl_err!("Compound identifier: {ids:?}");
- }
-
let search_result = search_dfschema(&ids, schema);
match search_result {
// found matching field with spare identifier(s) for nested
field(s) in structure
@@ -142,9 +135,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
}
}
}
- not_impl_err!(
- "Compound identifiers not supported by ExprPlanner:
{ids:?}"
- )
+ plan_err!("could not parse compound identifier from
{ids:?}")
}
// found matching field with no spare identifier(s)
Some((field, qualifier, _nested_names)) => {
diff --git a/datafusion/sqllogictest/test_files/struct.slt
b/datafusion/sqllogictest/test_files/struct.slt
index f3ac6549ad..d2e7160d00 100644
--- a/datafusion/sqllogictest/test_files/struct.slt
+++ b/datafusion/sqllogictest/test_files/struct.slt
@@ -282,3 +282,94 @@ drop table values;
statement ok
drop table struct_values;
+
+statement ok
+CREATE OR REPLACE VIEW complex_view AS
+SELECT {
+ 'user': {
+ 'info': {
+ 'personal': {
+ 'name': 'John Doe',
+ 'age': 30,
+ 'email': '[email protected]'
+ },
+ 'address': {
+ 'street': '123 Main St',
+ 'city': 'Anytown',
+ 'country': 'Countryland',
+ 'coordinates': [40.7128, -74.0060]
+ }
+ },
+ 'preferences': {
+ 'theme': 'dark',
+ 'notifications': true,
+ 'languages': ['en', 'es', 'fr']
+ },
+ 'stats': {
+ 'logins': 42,
+ 'last_active': '2023-09-15',
+ 'scores': [85, 92, 78, 95],
+ 'achievements': {
+ 'badges': ['early_bird', 'top_contributor'],
+ 'levels': {
+ 'beginner': true,
+ 'intermediate': true,
+ 'advanced': false
+ }
+ }
+ }
+ },
+ 'metadata': {
+ 'version': '1.0',
+ 'created_at': '2023-09-01T12:00:00Z'
+ },
+ 'deep_nested': {
+ 'level1': {
+ 'level2': {
+ 'level3': {
+ 'level4': {
+ 'level5': {
+ 'level6': {
+ 'level7': {
+ 'level8': {
+ 'level9': {
+ 'level10': 'You reached the
bottom!'
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+} AS complex_data;
+
+query T
+SELECT complex_data.user.info.personal.name FROM complex_view;
+----
+John Doe
+
+query I
+SELECT complex_data.user.info.personal.age FROM complex_view;
+----
+30
+
+query T
+SELECT complex_data.user.info.address.city FROM complex_view;
+----
+Anytown
+
+query T
+SELECT complex_data.user.preferences.languages[2] FROM complex_view;
+----
+es
+
+query T
+SELECT
complex_data.deep_nested.level1.level2.level3.level4.level5.level6.level7.level8.level9.level10
FROM complex_view;
+----
+You reached the bottom!
+
+statement ok
+drop view complex_view;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
