This is an automated email from the ASF dual-hosted git repository.
xudong963 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new a7bb09f069 move resolve_table_references` out of `datafusion-catalog`
(#14441)
a7bb09f069 is described below
commit a7bb09f069ece16a916565ccb971bd69d0c2bc27
Author: logan-keede <[email protected]>
AuthorDate: Mon Feb 3 21:27:21 2025 +0530
move resolve_table_references` out of `datafusion-catalog` (#14441)
* move out of
* forgotten license
* Update references
---------
Co-authored-by: Andrew Lamb <[email protected]>
---
datafusion-examples/examples/sql_frontend.rs | 3 +-
datafusion/catalog/src/lib.rs | 248 +-----------------
datafusion/core/tests/optimizer/mod.rs | 3 +-
.../optimizer/tests/optimizer_integration.rs | 3 +-
datafusion/sql/examples/sql.rs | 3 +-
datafusion/sql/src/expr/mod.rs | 3 +-
datafusion/sql/src/lib.rs | 6 +-
.../{catalog/src/lib.rs => sql/src/resolve.rs} | 281 ++++++++++-----------
datafusion/substrait/src/logical_plan/consumer.rs | 3 +-
9 files changed, 156 insertions(+), 397 deletions(-)
diff --git a/datafusion-examples/examples/sql_frontend.rs
b/datafusion-examples/examples/sql_frontend.rs
index c7afb79084..3955d5038c 100644
--- a/datafusion-examples/examples/sql_frontend.rs
+++ b/datafusion-examples/examples/sql_frontend.rs
@@ -16,7 +16,7 @@
// under the License.
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
-use datafusion::common::plan_err;
+use datafusion::common::{plan_err, TableReference};
use datafusion::config::ConfigOptions;
use datafusion::error::Result;
use datafusion::logical_expr::{
@@ -29,7 +29,6 @@ use datafusion::optimizer::{
use datafusion::sql::planner::{ContextProvider, SqlToRel};
use datafusion::sql::sqlparser::dialect::PostgreSqlDialect;
use datafusion::sql::sqlparser::parser::Parser;
-use datafusion::sql::TableReference;
use std::any::Any;
use std::sync::Arc;
diff --git a/datafusion/catalog/src/lib.rs b/datafusion/catalog/src/lib.rs
index 63d75fa3ef..0b8d73fabd 100644
--- a/datafusion/catalog/src/lib.rs
+++ b/datafusion/catalog/src/lib.rs
@@ -22,13 +22,19 @@
//! * Simple memory based catalog: [`MemoryCatalogProviderList`],
[`MemoryCatalogProvider`], [`MemorySchemaProvider`]
pub mod memory;
+#[deprecated(
+ since = "46.0.0",
+ note = "use datafusion_sql::resolve::resolve_table_references"
+)]
+pub use datafusion_sql::resolve::resolve_table_references;
+#[deprecated(
+ since = "46.0.0",
+ note = "use datafusion_common::{ResolvedTableReference, TableReference}"
+)]
pub use datafusion_sql::{ResolvedTableReference, TableReference};
pub use memory::{
MemoryCatalogProvider, MemoryCatalogProviderList, MemorySchemaProvider,
};
-use std::collections::BTreeSet;
-use std::ops::ControlFlow;
-
mod r#async;
mod catalog;
mod dynamic_file;
@@ -43,239 +49,3 @@ pub use schema::*;
pub use session::*;
pub use table::*;
pub mod streaming;
-
-/// Collects all tables and views referenced in the SQL statement. CTEs are
collected separately.
-/// This can be used to determine which tables need to be in the catalog for a
query to be planned.
-///
-/// # Returns
-///
-/// A `(table_refs, ctes)` tuple, the first element contains table and view
references and the second
-/// element contains any CTE aliases that were defined and possibly referenced.
-///
-/// ## Example
-///
-/// ```
-/// # use datafusion_sql::parser::DFParser;
-/// # use datafusion_catalog::resolve_table_references;
-/// let query = "SELECT a FROM foo where x IN (SELECT y FROM bar)";
-/// let statement = DFParser::parse_sql(query).unwrap().pop_back().unwrap();
-/// let (table_refs, ctes) = resolve_table_references(&statement,
true).unwrap();
-/// assert_eq!(table_refs.len(), 2);
-/// assert_eq!(table_refs[0].to_string(), "bar");
-/// assert_eq!(table_refs[1].to_string(), "foo");
-/// assert_eq!(ctes.len(), 0);
-/// ```
-///
-/// ## Example with CTEs
-///
-/// ```
-/// # use datafusion_sql::parser::DFParser;
-/// # use datafusion_catalog::resolve_table_references;
-/// let query = "with my_cte as (values (1), (2)) SELECT * from my_cte;";
-/// let statement = DFParser::parse_sql(query).unwrap().pop_back().unwrap();
-/// let (table_refs, ctes) = resolve_table_references(&statement,
true).unwrap();
-/// assert_eq!(table_refs.len(), 0);
-/// assert_eq!(ctes.len(), 1);
-/// assert_eq!(ctes[0].to_string(), "my_cte");
-/// ```
-pub fn resolve_table_references(
- statement: &datafusion_sql::parser::Statement,
- enable_ident_normalization: bool,
-) -> datafusion_common::Result<(Vec<TableReference>, Vec<TableReference>)> {
- use datafusion_sql::parser::{
- CopyToSource, CopyToStatement, Statement as DFStatement,
- };
- use datafusion_sql::planner::object_name_to_table_reference;
- use information_schema::INFORMATION_SCHEMA;
- use information_schema::INFORMATION_SCHEMA_TABLES;
- use sqlparser::ast::*;
-
- struct RelationVisitor {
- relations: BTreeSet<ObjectName>,
- all_ctes: BTreeSet<ObjectName>,
- ctes_in_scope: Vec<ObjectName>,
- }
-
- impl RelationVisitor {
- /// Record the reference to `relation`, if it's not a CTE reference.
- fn insert_relation(&mut self, relation: &ObjectName) {
- if !self.relations.contains(relation)
- && !self.ctes_in_scope.contains(relation)
- {
- self.relations.insert(relation.clone());
- }
- }
- }
-
- impl Visitor for RelationVisitor {
- type Break = ();
-
- fn pre_visit_relation(&mut self, relation: &ObjectName) ->
ControlFlow<()> {
- self.insert_relation(relation);
- ControlFlow::Continue(())
- }
-
- fn pre_visit_query(&mut self, q: &Query) -> ControlFlow<Self::Break> {
- if let Some(with) = &q.with {
- for cte in &with.cte_tables {
- // The non-recursive CTE name is not in scope when
evaluating the CTE itself, so this is valid:
- // `WITH t AS (SELECT * FROM t) SELECT * FROM t`
- // Where the first `t` refers to a predefined table. So we
are careful here
- // to visit the CTE first, before putting it in scope.
- if !with.recursive {
- // This is a bit hackish as the CTE will be visited
again as part of visiting `q`,
- // but thankfully `insert_relation` is idempotent.
- cte.visit(self);
- }
- self.ctes_in_scope
- .push(ObjectName(vec![cte.alias.name.clone()]));
- }
- }
- ControlFlow::Continue(())
- }
-
- fn post_visit_query(&mut self, q: &Query) -> ControlFlow<Self::Break> {
- if let Some(with) = &q.with {
- for _ in &with.cte_tables {
- // Unwrap: We just pushed these in `pre_visit_query`
- self.all_ctes.insert(self.ctes_in_scope.pop().unwrap());
- }
- }
- ControlFlow::Continue(())
- }
-
- fn pre_visit_statement(&mut self, statement: &Statement) ->
ControlFlow<()> {
- if let Statement::ShowCreate {
- obj_type: ShowCreateObject::Table | ShowCreateObject::View,
- obj_name,
- } = statement
- {
- self.insert_relation(obj_name)
- }
-
- // SHOW statements will later be rewritten into a SELECT from the
information_schema
- let requires_information_schema = matches!(
- statement,
- Statement::ShowFunctions { .. }
- | Statement::ShowVariable { .. }
- | Statement::ShowStatus { .. }
- | Statement::ShowVariables { .. }
- | Statement::ShowCreate { .. }
- | Statement::ShowColumns { .. }
- | Statement::ShowTables { .. }
- | Statement::ShowCollation { .. }
- );
- if requires_information_schema {
- for s in INFORMATION_SCHEMA_TABLES {
- self.relations.insert(ObjectName(vec![
- Ident::new(INFORMATION_SCHEMA),
- Ident::new(*s),
- ]));
- }
- }
- ControlFlow::Continue(())
- }
- }
-
- let mut visitor = RelationVisitor {
- relations: BTreeSet::new(),
- all_ctes: BTreeSet::new(),
- ctes_in_scope: vec![],
- };
-
- fn visit_statement(statement: &DFStatement, visitor: &mut RelationVisitor)
{
- match statement {
- DFStatement::Statement(s) => {
- let _ = s.as_ref().visit(visitor);
- }
- DFStatement::CreateExternalTable(table) => {
- visitor.relations.insert(table.name.clone());
- }
- DFStatement::CopyTo(CopyToStatement { source, .. }) => match
source {
- CopyToSource::Relation(table_name) => {
- visitor.insert_relation(table_name);
- }
- CopyToSource::Query(query) => {
- query.visit(visitor);
- }
- },
- DFStatement::Explain(explain) =>
visit_statement(&explain.statement, visitor),
- }
- }
-
- visit_statement(statement, &mut visitor);
-
- let table_refs = visitor
- .relations
- .into_iter()
- .map(|x| object_name_to_table_reference(x, enable_ident_normalization))
- .collect::<datafusion_common::Result<_>>()?;
- let ctes = visitor
- .all_ctes
- .into_iter()
- .map(|x| object_name_to_table_reference(x, enable_ident_normalization))
- .collect::<datafusion_common::Result<_>>()?;
- Ok((table_refs, ctes))
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn resolve_table_references_shadowed_cte() {
- use datafusion_sql::parser::DFParser;
-
- // An interesting edge case where the `t` name is used both as an
ordinary table reference
- // and as a CTE reference.
- let query = "WITH t AS (SELECT * FROM t) SELECT * FROM t";
- let statement =
DFParser::parse_sql(query).unwrap().pop_back().unwrap();
- let (table_refs, ctes) = resolve_table_references(&statement,
true).unwrap();
- assert_eq!(table_refs.len(), 1);
- assert_eq!(ctes.len(), 1);
- assert_eq!(ctes[0].to_string(), "t");
- assert_eq!(table_refs[0].to_string(), "t");
-
- // UNION is a special case where the CTE is not in scope for the
second branch.
- let query = "(with t as (select 1) select * from t) union (select *
from t)";
- let statement =
DFParser::parse_sql(query).unwrap().pop_back().unwrap();
- let (table_refs, ctes) = resolve_table_references(&statement,
true).unwrap();
- assert_eq!(table_refs.len(), 1);
- assert_eq!(ctes.len(), 1);
- assert_eq!(ctes[0].to_string(), "t");
- assert_eq!(table_refs[0].to_string(), "t");
-
- // Nested CTEs are also handled.
- // Here the first `u` is a CTE, but the second `u` is a table
reference.
- // While `t` is always a CTE.
- let query = "(with t as (with u as (select 1) select * from u) select
* from u cross join t)";
- let statement =
DFParser::parse_sql(query).unwrap().pop_back().unwrap();
- let (table_refs, ctes) = resolve_table_references(&statement,
true).unwrap();
- assert_eq!(table_refs.len(), 1);
- assert_eq!(ctes.len(), 2);
- assert_eq!(ctes[0].to_string(), "t");
- assert_eq!(ctes[1].to_string(), "u");
- assert_eq!(table_refs[0].to_string(), "u");
- }
-
- #[test]
- fn resolve_table_references_recursive_cte() {
- use datafusion_sql::parser::DFParser;
-
- let query = "
- WITH RECURSIVE nodes AS (
- SELECT 1 as id
- UNION ALL
- SELECT id + 1 as id
- FROM nodes
- WHERE id < 10
- )
- SELECT * FROM nodes
- ";
- let statement =
DFParser::parse_sql(query).unwrap().pop_back().unwrap();
- let (table_refs, ctes) = resolve_table_references(&statement,
true).unwrap();
- assert_eq!(table_refs.len(), 0);
- assert_eq!(ctes.len(), 1);
- assert_eq!(ctes[0].to_string(), "nodes");
- }
-}
diff --git a/datafusion/core/tests/optimizer/mod.rs
b/datafusion/core/tests/optimizer/mod.rs
index f17d13a420..37a6ca7f59 100644
--- a/datafusion/core/tests/optimizer/mod.rs
+++ b/datafusion/core/tests/optimizer/mod.rs
@@ -26,7 +26,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef,
TimeUnit};
use arrow_schema::{Fields, SchemaBuilder};
use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{TransformedResult, TreeNode};
-use datafusion_common::{plan_err, DFSchema, Result, ScalarValue};
+use datafusion_common::{plan_err, DFSchema, Result, ScalarValue,
TableReference};
use datafusion_expr::interval_arithmetic::{Interval, NullableInterval};
use datafusion_expr::{
col, lit, AggregateUDF, BinaryExpr, Expr, ExprSchemable, LogicalPlan,
Operator,
@@ -41,7 +41,6 @@ use datafusion_sql::planner::{ContextProvider, SqlToRel};
use datafusion_sql::sqlparser::ast::Statement;
use datafusion_sql::sqlparser::dialect::GenericDialect;
use datafusion_sql::sqlparser::parser::Parser;
-use datafusion_sql::TableReference;
use chrono::DateTime;
use datafusion_functions::datetime;
diff --git a/datafusion/optimizer/tests/optimizer_integration.rs
b/datafusion/optimizer/tests/optimizer_integration.rs
index b9073f5ac8..a33ecbc3a1 100644
--- a/datafusion/optimizer/tests/optimizer_integration.rs
+++ b/datafusion/optimizer/tests/optimizer_integration.rs
@@ -22,7 +22,7 @@ use std::sync::Arc;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit};
use datafusion_common::config::ConfigOptions;
-use datafusion_common::{assert_contains, plan_err, Result};
+use datafusion_common::{assert_contains, plan_err, Result, TableReference};
use datafusion_expr::sqlparser::dialect::PostgreSqlDialect;
use datafusion_expr::test::function_stub::sum_udaf;
use datafusion_expr::{AggregateUDF, LogicalPlan, ScalarUDF, TableSource,
WindowUDF};
@@ -36,7 +36,6 @@ use datafusion_sql::planner::{ContextProvider, SqlToRel};
use datafusion_sql::sqlparser::ast::Statement;
use datafusion_sql::sqlparser::dialect::GenericDialect;
use datafusion_sql::sqlparser::parser::Parser;
-use datafusion_sql::TableReference;
#[cfg(test)]
#[ctor::ctor]
diff --git a/datafusion/sql/examples/sql.rs b/datafusion/sql/examples/sql.rs
index aa17be6273..7f1e6bf8f2 100644
--- a/datafusion/sql/examples/sql.rs
+++ b/datafusion/sql/examples/sql.rs
@@ -20,7 +20,7 @@ use std::{collections::HashMap, sync::Arc};
use arrow_schema::{DataType, Field, Schema};
use datafusion_common::config::ConfigOptions;
-use datafusion_common::{plan_err, Result};
+use datafusion_common::{plan_err, Result, TableReference};
use datafusion_expr::planner::ExprPlanner;
use datafusion_expr::WindowUDF;
use datafusion_expr::{
@@ -32,7 +32,6 @@ use datafusion_functions_aggregate::sum::sum_udaf;
use datafusion_sql::{
planner::{ContextProvider, SqlToRel},
sqlparser::{dialect::GenericDialect, parser::Parser},
- TableReference,
};
fn main() {
diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs
index eea64b9e8e..951e81c1fd 100644
--- a/datafusion/sql/src/expr/mod.rs
+++ b/datafusion/sql/src/expr/mod.rs
@@ -1073,11 +1073,10 @@ mod tests {
use sqlparser::parser::Parser;
use datafusion_common::config::ConfigOptions;
+ use datafusion_common::TableReference;
use datafusion_expr::logical_plan::builder::LogicalTableSource;
use datafusion_expr::{AggregateUDF, ScalarUDF, TableSource, WindowUDF};
- use crate::TableReference;
-
use super::*;
struct TestContextProvider {
diff --git a/datafusion/sql/src/lib.rs b/datafusion/sql/src/lib.rs
index f20560fe7c..16a3d6d007 100644
--- a/datafusion/sql/src/lib.rs
+++ b/datafusion/sql/src/lib.rs
@@ -41,6 +41,7 @@ pub mod parser;
pub mod planner;
mod query;
mod relation;
+pub mod resolve;
mod select;
mod set_expr;
mod stack;
@@ -49,6 +50,9 @@ mod statement;
pub mod unparser;
pub mod utils;
mod values;
-
+#[deprecated(
+ since = "46.0.0",
+ note = "use datafusion_common::{ResolvedTableReference, TableReference}"
+)]
pub use datafusion_common::{ResolvedTableReference, TableReference};
pub use sqlparser;
diff --git a/datafusion/catalog/src/lib.rs b/datafusion/sql/src/resolve.rs
similarity index 51%
copy from datafusion/catalog/src/lib.rs
copy to datafusion/sql/src/resolve.rs
index 63d75fa3ef..88416dfe03 100644
--- a/datafusion/catalog/src/lib.rs
+++ b/datafusion/sql/src/resolve.rs
@@ -15,34 +15,140 @@
// specific language governing permissions and limitations
// under the License.
-//! Interfaces and default implementations of catalogs and schemas.
-//!
-//! Implementations
-//! * Information schema: [`information_schema`]
-//! * Simple memory based catalog: [`MemoryCatalogProviderList`],
[`MemoryCatalogProvider`], [`MemorySchemaProvider`]
-
-pub mod memory;
-pub use datafusion_sql::{ResolvedTableReference, TableReference};
-pub use memory::{
- MemoryCatalogProvider, MemoryCatalogProviderList, MemorySchemaProvider,
-};
+use crate::TableReference;
use std::collections::BTreeSet;
use std::ops::ControlFlow;
-mod r#async;
-mod catalog;
-mod dynamic_file;
-pub mod information_schema;
-mod schema;
-mod session;
-mod table;
-pub use catalog::*;
-pub use dynamic_file::catalog::*;
-pub use r#async::*;
-pub use schema::*;
-pub use session::*;
-pub use table::*;
-pub mod streaming;
+use crate::parser::{CopyToSource, CopyToStatement, Statement as DFStatement};
+use crate::planner::object_name_to_table_reference;
+use sqlparser::ast::*;
+
+// following constants are used in `resolve_table_references`
+// and should be same as `datafusion/catalog/src/information_schema.rs`
+const INFORMATION_SCHEMA: &str = "information_schema";
+const TABLES: &str = "tables";
+const VIEWS: &str = "views";
+const COLUMNS: &str = "columns";
+const DF_SETTINGS: &str = "df_settings";
+const SCHEMATA: &str = "schemata";
+const ROUTINES: &str = "routines";
+const PARAMETERS: &str = "parameters";
+
+/// All information schema tables
+const INFORMATION_SCHEMA_TABLES: &[&str] = &[
+ TABLES,
+ VIEWS,
+ COLUMNS,
+ DF_SETTINGS,
+ SCHEMATA,
+ ROUTINES,
+ PARAMETERS,
+];
+
+struct RelationVisitor {
+ relations: BTreeSet<ObjectName>,
+ all_ctes: BTreeSet<ObjectName>,
+ ctes_in_scope: Vec<ObjectName>,
+}
+
+impl RelationVisitor {
+ /// Record the reference to `relation`, if it's not a CTE reference.
+ fn insert_relation(&mut self, relation: &ObjectName) {
+ if !self.relations.contains(relation) &&
!self.ctes_in_scope.contains(relation) {
+ self.relations.insert(relation.clone());
+ }
+ }
+}
+
+impl Visitor for RelationVisitor {
+ type Break = ();
+
+ fn pre_visit_relation(&mut self, relation: &ObjectName) -> ControlFlow<()>
{
+ self.insert_relation(relation);
+ ControlFlow::Continue(())
+ }
+
+ fn pre_visit_query(&mut self, q: &Query) -> ControlFlow<Self::Break> {
+ if let Some(with) = &q.with {
+ for cte in &with.cte_tables {
+ // The non-recursive CTE name is not in scope when evaluating
the CTE itself, so this is valid:
+ // `WITH t AS (SELECT * FROM t) SELECT * FROM t`
+ // Where the first `t` refers to a predefined table. So we are
careful here
+ // to visit the CTE first, before putting it in scope.
+ if !with.recursive {
+ // This is a bit hackish as the CTE will be visited again
as part of visiting `q`,
+ // but thankfully `insert_relation` is idempotent.
+ cte.visit(self);
+ }
+ self.ctes_in_scope
+ .push(ObjectName(vec![cte.alias.name.clone()]));
+ }
+ }
+ ControlFlow::Continue(())
+ }
+
+ fn post_visit_query(&mut self, q: &Query) -> ControlFlow<Self::Break> {
+ if let Some(with) = &q.with {
+ for _ in &with.cte_tables {
+ // Unwrap: We just pushed these in `pre_visit_query`
+ self.all_ctes.insert(self.ctes_in_scope.pop().unwrap());
+ }
+ }
+ ControlFlow::Continue(())
+ }
+
+ fn pre_visit_statement(&mut self, statement: &Statement) ->
ControlFlow<()> {
+ if let Statement::ShowCreate {
+ obj_type: ShowCreateObject::Table | ShowCreateObject::View,
+ obj_name,
+ } = statement
+ {
+ self.insert_relation(obj_name)
+ }
+
+ // SHOW statements will later be rewritten into a SELECT from the
information_schema
+ let requires_information_schema = matches!(
+ statement,
+ Statement::ShowFunctions { .. }
+ | Statement::ShowVariable { .. }
+ | Statement::ShowStatus { .. }
+ | Statement::ShowVariables { .. }
+ | Statement::ShowCreate { .. }
+ | Statement::ShowColumns { .. }
+ | Statement::ShowTables { .. }
+ | Statement::ShowCollation { .. }
+ );
+ if requires_information_schema {
+ for s in INFORMATION_SCHEMA_TABLES {
+ self.relations.insert(ObjectName(vec![
+ Ident::new(INFORMATION_SCHEMA),
+ Ident::new(*s),
+ ]));
+ }
+ }
+ ControlFlow::Continue(())
+ }
+}
+
+fn visit_statement(statement: &DFStatement, visitor: &mut RelationVisitor) {
+ match statement {
+ DFStatement::Statement(s) => {
+ let _ = s.as_ref().visit(visitor);
+ }
+ DFStatement::CreateExternalTable(table) => {
+ visitor.relations.insert(table.name.clone());
+ }
+ DFStatement::CopyTo(CopyToStatement { source, .. }) => match source {
+ CopyToSource::Relation(table_name) => {
+ visitor.insert_relation(table_name);
+ }
+ CopyToSource::Query(query) => {
+ query.visit(visitor);
+ }
+ },
+ DFStatement::Explain(explain) => visit_statement(&explain.statement,
visitor),
+ }
+}
/// Collects all tables and views referenced in the SQL statement. CTEs are
collected separately.
/// This can be used to determine which tables need to be in the catalog for a
query to be planned.
@@ -56,7 +162,7 @@ pub mod streaming;
///
/// ```
/// # use datafusion_sql::parser::DFParser;
-/// # use datafusion_catalog::resolve_table_references;
+/// # use datafusion_sql::resolve::resolve_table_references;
/// let query = "SELECT a FROM foo where x IN (SELECT y FROM bar)";
/// let statement = DFParser::parse_sql(query).unwrap().pop_back().unwrap();
/// let (table_refs, ctes) = resolve_table_references(&statement,
true).unwrap();
@@ -70,7 +176,7 @@ pub mod streaming;
///
/// ```
/// # use datafusion_sql::parser::DFParser;
-/// # use datafusion_catalog::resolve_table_references;
+/// # use datafusion_sql::resolve::resolve_table_references;
/// let query = "with my_cte as (values (1), (2)) SELECT * from my_cte;";
/// let statement = DFParser::parse_sql(query).unwrap().pop_back().unwrap();
/// let (table_refs, ctes) = resolve_table_references(&statement,
true).unwrap();
@@ -79,130 +185,15 @@ pub mod streaming;
/// assert_eq!(ctes[0].to_string(), "my_cte");
/// ```
pub fn resolve_table_references(
- statement: &datafusion_sql::parser::Statement,
+ statement: &crate::parser::Statement,
enable_ident_normalization: bool,
) -> datafusion_common::Result<(Vec<TableReference>, Vec<TableReference>)> {
- use datafusion_sql::parser::{
- CopyToSource, CopyToStatement, Statement as DFStatement,
- };
- use datafusion_sql::planner::object_name_to_table_reference;
- use information_schema::INFORMATION_SCHEMA;
- use information_schema::INFORMATION_SCHEMA_TABLES;
- use sqlparser::ast::*;
-
- struct RelationVisitor {
- relations: BTreeSet<ObjectName>,
- all_ctes: BTreeSet<ObjectName>,
- ctes_in_scope: Vec<ObjectName>,
- }
-
- impl RelationVisitor {
- /// Record the reference to `relation`, if it's not a CTE reference.
- fn insert_relation(&mut self, relation: &ObjectName) {
- if !self.relations.contains(relation)
- && !self.ctes_in_scope.contains(relation)
- {
- self.relations.insert(relation.clone());
- }
- }
- }
-
- impl Visitor for RelationVisitor {
- type Break = ();
-
- fn pre_visit_relation(&mut self, relation: &ObjectName) ->
ControlFlow<()> {
- self.insert_relation(relation);
- ControlFlow::Continue(())
- }
-
- fn pre_visit_query(&mut self, q: &Query) -> ControlFlow<Self::Break> {
- if let Some(with) = &q.with {
- for cte in &with.cte_tables {
- // The non-recursive CTE name is not in scope when
evaluating the CTE itself, so this is valid:
- // `WITH t AS (SELECT * FROM t) SELECT * FROM t`
- // Where the first `t` refers to a predefined table. So we
are careful here
- // to visit the CTE first, before putting it in scope.
- if !with.recursive {
- // This is a bit hackish as the CTE will be visited
again as part of visiting `q`,
- // but thankfully `insert_relation` is idempotent.
- cte.visit(self);
- }
- self.ctes_in_scope
- .push(ObjectName(vec![cte.alias.name.clone()]));
- }
- }
- ControlFlow::Continue(())
- }
-
- fn post_visit_query(&mut self, q: &Query) -> ControlFlow<Self::Break> {
- if let Some(with) = &q.with {
- for _ in &with.cte_tables {
- // Unwrap: We just pushed these in `pre_visit_query`
- self.all_ctes.insert(self.ctes_in_scope.pop().unwrap());
- }
- }
- ControlFlow::Continue(())
- }
-
- fn pre_visit_statement(&mut self, statement: &Statement) ->
ControlFlow<()> {
- if let Statement::ShowCreate {
- obj_type: ShowCreateObject::Table | ShowCreateObject::View,
- obj_name,
- } = statement
- {
- self.insert_relation(obj_name)
- }
-
- // SHOW statements will later be rewritten into a SELECT from the
information_schema
- let requires_information_schema = matches!(
- statement,
- Statement::ShowFunctions { .. }
- | Statement::ShowVariable { .. }
- | Statement::ShowStatus { .. }
- | Statement::ShowVariables { .. }
- | Statement::ShowCreate { .. }
- | Statement::ShowColumns { .. }
- | Statement::ShowTables { .. }
- | Statement::ShowCollation { .. }
- );
- if requires_information_schema {
- for s in INFORMATION_SCHEMA_TABLES {
- self.relations.insert(ObjectName(vec![
- Ident::new(INFORMATION_SCHEMA),
- Ident::new(*s),
- ]));
- }
- }
- ControlFlow::Continue(())
- }
- }
-
let mut visitor = RelationVisitor {
relations: BTreeSet::new(),
all_ctes: BTreeSet::new(),
ctes_in_scope: vec![],
};
- fn visit_statement(statement: &DFStatement, visitor: &mut RelationVisitor)
{
- match statement {
- DFStatement::Statement(s) => {
- let _ = s.as_ref().visit(visitor);
- }
- DFStatement::CreateExternalTable(table) => {
- visitor.relations.insert(table.name.clone());
- }
- DFStatement::CopyTo(CopyToStatement { source, .. }) => match
source {
- CopyToSource::Relation(table_name) => {
- visitor.insert_relation(table_name);
- }
- CopyToSource::Query(query) => {
- query.visit(visitor);
- }
- },
- DFStatement::Explain(explain) =>
visit_statement(&explain.statement, visitor),
- }
- }
-
visit_statement(statement, &mut visitor);
let table_refs = visitor
@@ -224,7 +215,7 @@ mod tests {
#[test]
fn resolve_table_references_shadowed_cte() {
- use datafusion_sql::parser::DFParser;
+ use crate::parser::DFParser;
// An interesting edge case where the `t` name is used both as an
ordinary table reference
// and as a CTE reference.
@@ -260,7 +251,7 @@ mod tests {
#[test]
fn resolve_table_references_recursive_cte() {
- use datafusion_sql::parser::DFParser;
+ use crate::parser::DFParser;
let query = "
WITH RECURSIVE nodes AS (
diff --git a/datafusion/substrait/src/logical_plan/consumer.rs
b/datafusion/substrait/src/logical_plan/consumer.rs
index 5a7d70c5e7..b17a8967e5 100644
--- a/datafusion/substrait/src/logical_plan/consumer.rs
+++ b/datafusion/substrait/src/logical_plan/consumer.rs
@@ -23,7 +23,7 @@ use datafusion::arrow::datatypes::{
};
use datafusion::common::{
not_impl_datafusion_err, not_impl_err, plan_datafusion_err, plan_err,
- substrait_datafusion_err, substrait_err, DFSchema, DFSchemaRef,
+ substrait_datafusion_err, substrait_err, DFSchema, DFSchemaRef,
TableReference,
};
use datafusion::datasource::provider_as_source;
use datafusion::logical_expr::expr::{Exists, InSubquery, Sort};
@@ -66,7 +66,6 @@ use datafusion::logical_expr::{
WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition,
};
use datafusion::prelude::{lit, JoinType};
-use datafusion::sql::TableReference;
use datafusion::{
error::Result, logical_expr::utils::split_conjunction, prelude::Column,
scalar::ScalarValue,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
