This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-22290-c8b784a01f5d0bcbe0dac806730fb61afc0be8ef in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 7bf5fc2148c53cd39f060bbc23d521a66c4894cf Author: Kumar Ujjawal <[email protected]> AuthorDate: Tue May 19 21:03:34 2026 +0530 fix(sql): reject duplicate unqualified names in CTAS, CREATE VIEW, and SELECT INTO (#22290) ## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes #22167 ## Rationale for this change `CREATE TABLE AS`, `CREATE VIEW`, and `SELECT INTO` can persist only plain column names. Before this change, a query like a self join could produce qualified output columns such as `x.column1` and `y.column1`, which are valid while planning. But when the table or view was created, those qualifiers were dropped and the stored schema became `column1`, `column1`, ... with duplicates. That meant the create step succeeded, but the created table or view failed later when the user tried to read from it. The object existed in the catalog but was not usable. This PR fixes that by rejecting these cases during execution of the DDL operators, before the table or view is registered. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> ## What changes are included in this PR? This PR adds an execution-side check that validates the final output schema has unique unqualified column names. It applies that check in: - `create_memory_table` - `create_view` This covers: - `CREATE TABLE AS` - `CREATE VIEW` - `SELECT INTO` The check runs after optimization or type coercion and before registration, so it validates the actual names that would be stored. For `IF NOT EXISTS`, execution keeps the existing no-op behavior when the target already exists, so validation only runs when a new object would actually be created or replaced. This PR also adds: - sqllogictest coverage for the duplicate-column self-join case for all three flows - direct execution-level regression tests for create / replace table and view paths <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? Yes <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? Invalid `CREATE TABLE AS`, `CREATE VIEW`, and `SELECT INTO` statements that would create an object with duplicate unqualified column names now fail early during DDL execution with a schema error, instead of succeeding and creating an unreadable table or view. No API Changes <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> --- datafusion/core/src/execution/context/mod.rs | 25 +++- datafusion/core/tests/dataframe/mod.rs | 172 ++++++++++++++++++++++++++- datafusion/sqllogictest/test_files/ddl.slt | 20 ++++ 3 files changed, 210 insertions(+), 7 deletions(-) diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index 72d6c4c3a6..5770c20262 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -67,10 +67,10 @@ use datafusion_catalog::{ use datafusion_common::config::{ConfigField, ConfigOptions}; use datafusion_common::metadata::ScalarAndMetadata; use datafusion_common::{ - DFSchema, DataFusionError, ParamValues, SchemaReference, TableReference, + DFSchema, DataFusionError, ParamValues, SchemaError, SchemaReference, TableReference, config::{ConfigExtension, TableOptions}, exec_datafusion_err, exec_err, internal_datafusion_err, not_impl_err, - plan_datafusion_err, plan_err, + plan_datafusion_err, plan_err, schema_err, tree_node::{TreeNodeRecursion, TreeNodeVisitor}, }; pub use datafusion_execution::TaskContext; @@ -887,6 +887,7 @@ impl SessionContext { match (if_not_exists, or_replace, table) { (true, false, Ok(_)) => self.return_empty_dataframe(), (false, true, Ok(_)) => { + Self::ensure_unique_column_names(input.schema())?; self.deregister_table(name.clone())?; let schema = Arc::clone(input.schema().inner()); let physical = DataFrame::new(self.state(), input); @@ -906,6 +907,7 @@ impl SessionContext { exec_err!("'IF NOT EXISTS' cannot coexist with 'REPLACE'") } (_, _, Err(_)) => { + Self::ensure_unique_column_names(input.schema())?; let schema = Arc::clone(input.schema().inner()); let physical = DataFrame::new(self.state(), input); @@ -951,14 +953,16 @@ impl SessionContext { match (or_replace, view) { (true, Ok(_)) => { - self.deregister_table(name.clone())?; let input = Self::apply_type_coercion(Arc::unwrap_or_clone(input))?; + Self::ensure_unique_column_names(input.schema())?; + self.deregister_table(name.clone())?; let table = Arc::new(ViewTable::new(input, definition)); self.register_table(name, table)?; self.return_empty_dataframe() } (_, Err(_)) => { let input = Self::apply_type_coercion(Arc::unwrap_or_clone(input))?; + Self::ensure_unique_column_names(input.schema())?; let table = Arc::new(ViewTable::new(input, definition)); self.register_table(name, table)?; self.return_empty_dataframe() @@ -967,6 +971,21 @@ impl SessionContext { } } + fn ensure_unique_column_names(schema: &DFSchema) -> Result<()> { + // DFSchema name checks allow duplicate unqualified names as long as their + // qualifiers differ. DDL persistence drops qualifiers, so this helper must + // enforce uniqueness on the final unqualified names instead. + let mut seen = HashSet::with_capacity(schema.fields().len()); + for field in schema.fields() { + if !seen.insert(field.name().as_str()) { + return schema_err!(SchemaError::DuplicateUnqualifiedField { + name: field.name().to_string(), + }); + } + } + Ok(()) + } + async fn create_catalog_schema(&self, cmd: CreateCatalogSchema) -> Result<DataFrame> { let CreateCatalogSchema { schema_name, diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs index d1dc964da8..e55a373ada 100644 --- a/datafusion/core/tests/dataframe/mod.rs +++ b/datafusion/core/tests/dataframe/mod.rs @@ -75,10 +75,11 @@ use datafusion_execution::runtime_env::RuntimeEnv; use datafusion_expr::expr::{GroupingSet, NullTreatment, Sort, WindowFunction}; use datafusion_expr::var_provider::{VarProvider, VarType}; use datafusion_expr::{ - Expr, ExprFunctionExt, ExprSchemable, LogicalPlan, LogicalPlanBuilder, - ScalarFunctionImplementation, SortExpr, TableType, WindowFrame, WindowFrameBound, - WindowFrameUnits, WindowFunctionDefinition, cast, col, create_udf, exists, - in_subquery, lit, out_ref_col, placeholder, scalar_subquery, when, wildcard, + CreateMemoryTable, CreateView, DdlStatement, Expr, ExprFunctionExt, ExprSchemable, + LogicalPlan, LogicalPlanBuilder, ScalarFunctionImplementation, SortExpr, TableType, + WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, cast, col, + create_udf, exists, in_subquery, lit, out_ref_col, placeholder, scalar_subquery, + when, wildcard, }; use datafusion_physical_expr::Partitioning; use datafusion_physical_expr::aggregate::AggregateExprBuilder; @@ -6750,6 +6751,169 @@ async fn test_copy_to_preserves_order() -> Result<()> { Ok(()) } +fn duplicate_unqualified_name_batch() -> Result<RecordBatch> { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Utf8, false), + Field::new("b", DataType::Int32, false), + ])); + + RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(vec![ + "abcDEF", + "abc123", + "CBAdef", + "123AbcDef", + ])), + Arc::new(Int32Array::from(vec![1, 10, 10, 100])), + ], + ) + .map_err(Into::into) +} + +async fn duplicate_unqualified_name_input( + ctx: &SessionContext, + name: &str, +) -> Result<LogicalPlan> { + ctx.register_batch(name, duplicate_unqualified_name_batch()?)?; + + let left = ctx.table(name).await?; + let right = left.clone().alias("t2")?; + Ok(left + .join(right, JoinType::Inner, &["b"], &["b"], None)? + .into_unoptimized_plan()) +} + +#[tokio::test] +async fn execute_logical_plan_rejects_duplicate_unqualified_names_in_create_memory_table() +-> Result<()> { + let ctx = SessionContext::new(); + let input = duplicate_unqualified_name_input(&ctx, "t1").await?; + + let plan = LogicalPlan::Ddl(DdlStatement::CreateMemoryTable(CreateMemoryTable { + name: TableReference::bare("dup_out"), + constraints: Constraints::default(), + input: Arc::new(input), + if_not_exists: false, + or_replace: false, + temporary: false, + column_defaults: vec![], + })); + + let err = ctx.execute_logical_plan(plan).await.unwrap_err(); + assert_contains!( + err.to_string(), + "Schema contains duplicate unqualified field name a" + ); + assert!(ctx.table("dup_out").await.is_err()); + + Ok(()) +} + +#[tokio::test] +async fn execute_logical_plan_rejects_duplicate_unqualified_names_in_replace_table() +-> Result<()> { + let ctx = SessionContext::new(); + ctx.sql("CREATE TABLE dup_out AS VALUES ('seed', 0)") + .await? + .collect() + .await?; + + let input = duplicate_unqualified_name_input(&ctx, "t1").await?; + let plan = LogicalPlan::Ddl(DdlStatement::CreateMemoryTable(CreateMemoryTable { + name: TableReference::bare("dup_out"), + constraints: Constraints::default(), + input: Arc::new(input), + if_not_exists: false, + or_replace: true, + temporary: false, + column_defaults: vec![], + })); + + let err = ctx.execute_logical_plan(plan).await.unwrap_err(); + assert_contains!( + err.to_string(), + "Schema contains duplicate unqualified field name a" + ); + + let rows = ctx.sql("SELECT * FROM dup_out").await?.collect().await?; + assert_batches_eq!( + [ + "+---------+---------+", + "| column1 | column2 |", + "+---------+---------+", + "| seed | 0 |", + "+---------+---------+", + ], + &rows + ); + + Ok(()) +} + +#[tokio::test] +async fn execute_logical_plan_rejects_duplicate_unqualified_names_in_create_view() +-> Result<()> { + let ctx = SessionContext::new(); + let input = duplicate_unqualified_name_input(&ctx, "t1").await?; + let plan = LogicalPlan::Ddl(DdlStatement::CreateView(CreateView { + name: TableReference::bare("dup_view"), + input: Arc::new(input), + or_replace: false, + definition: None, + temporary: false, + })); + + let err = ctx.execute_logical_plan(plan).await.unwrap_err(); + assert_contains!( + err.to_string(), + "Schema contains duplicate unqualified field name a" + ); + assert!(ctx.table("dup_view").await.is_err()); + + Ok(()) +} + +#[tokio::test] +async fn execute_logical_plan_rejects_duplicate_unqualified_names_in_replace_view() +-> Result<()> { + let ctx = SessionContext::new(); + ctx.sql("CREATE VIEW dup_view AS SELECT 'seed' AS column1, 0 AS column2") + .await? + .collect() + .await?; + + let input = duplicate_unqualified_name_input(&ctx, "t1").await?; + let plan = LogicalPlan::Ddl(DdlStatement::CreateView(CreateView { + name: TableReference::bare("dup_view"), + input: Arc::new(input), + or_replace: true, + definition: None, + temporary: false, + })); + + let err = ctx.execute_logical_plan(plan).await.unwrap_err(); + assert_contains!( + err.to_string(), + "Schema contains duplicate unqualified field name a" + ); + + let rows = ctx.sql("SELECT * FROM dup_view").await?.collect().await?; + assert_batches_eq!( + [ + "+---------+---------+", + "| column1 | column2 |", + "+---------+---------+", + "| seed | 0 |", + "+---------+---------+", + ], + &rows + ); + + Ok(()) +} + #[tokio::test] async fn test_duplicate_state_fields_for_dfschema_construct() -> Result<()> { let ctx = SessionContext::new(); diff --git a/datafusion/sqllogictest/test_files/ddl.slt b/datafusion/sqllogictest/test_files/ddl.slt index 977d2d03a1..82c30e9aba 100644 --- a/datafusion/sqllogictest/test_files/ddl.slt +++ b/datafusion/sqllogictest/test_files/ddl.slt @@ -971,3 +971,23 @@ drop view v; statement count 0 drop table t; + +# https://github.com/apache/datafusion/issues/22167 +# https://github.com/apache/datafusion/issues/13487 +# CREATE TABLE AS / CREATE VIEW must reject queries whose result schema +# has duplicate (unqualified) column names, since the resulting object +# stores only unqualified names and would otherwise be unselectable. +statement ok +CREATE TABLE dup_src AS VALUES(1, 2); + +statement error DataFusion error: Schema error: Schema contains duplicate unqualified field name column1 +CREATE TABLE dup_ctas AS SELECT * FROM dup_src LEFT JOIN dup_src y ON dup_src.column1 = y.column2; + +statement error DataFusion error: Schema error: Schema contains duplicate unqualified field name column1 +CREATE VIEW dup_view AS SELECT * FROM dup_src LEFT JOIN dup_src y ON dup_src.column1 = y.column2; + +statement error DataFusion error: Schema error: Schema contains duplicate unqualified field name column1 +SELECT * INTO dup_into FROM dup_src LEFT JOIN dup_src y ON dup_src.column1 = y.column2; + +statement ok +DROP TABLE dup_src; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
