JingsongLi commented on code in PR #1448: URL: https://github.com/apache/paimon-rust/pull/1448#discussion_r4657159807
########## crates/integrations/datafusion/src/sql_context/ctas.rs: ########## @@ -0,0 +1,285 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Persistent CREATE TABLE AS SELECT. Population failures retain the table +//! because concurrent writers can use it as soon as creation succeeds. + +use std::ops::ControlFlow; +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::execution::context::SQLOptions; +use datafusion::execution::session_state::SessionState; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::{execute_stream, ExecutionPlan}; +use datafusion::prelude::DataFrame; +use datafusion::sql::planner::object_name_to_table_reference; +use datafusion::sql::sqlparser::ast::helpers::stmt_create_table::CreateTableBuilder; +use datafusion::sql::sqlparser::ast::{ + CreateTable, CreateTableOptions, Query, Select, SqlOption, Statement, Visit, Visitor, +}; +use futures::StreamExt; +use paimon::spec::{CoreOptions, DataType as PaimonDataType, TableType}; +use paimon::table::Table; + +use super::{create_table_schema, extract_options, ok_result, SQLContext, TimeTravelSource}; +use crate::error::to_datafusion_error; +use crate::physical_plan::cast_batch_to_schema; + +impl SQLContext { + pub(super) async fn handle_create_table_as( + &self, + ct: &CreateTable, + query: Box<Query>, + partition_keys: Vec<String>, + state: SessionState, + options: SQLOptions, + time_travel_sources: &[TimeTravelSource], + ) -> DFResult<DataFrame> { + validate_create_table_as(ct, &query)?; + Self::ensure_main_branch_base_table(&ct.name, "CREATE TABLE AS SELECT")?; + let enable_ident_normalization = + state.config_options().sql_parser.enable_ident_normalization; + let target = object_name_to_table_reference(ct.name.clone(), enable_ident_normalization)?; + let (catalog, catalog_name, identifier) = + self.resolve_table_reference(&target, &state.config_options().catalog)?; + + match catalog.check_table_exists(&identifier).await { + Ok(()) if ct.if_not_exists => return ok_result(&self.ctx), + Ok(()) => { + return Err(to_datafusion_error(paimon::Error::TableAlreadyExist { + full_name: identifier.full_name(), + })); + } + Err(paimon::Error::TableNotExist { .. }) => {} + Err(error) => return Err(to_datafusion_error(error)), + } + + let table_options = extract_options(&ct.table_options)?; + let table_type = CoreOptions::new(&table_options) + .table_type() + .map_err(to_datafusion_error)?; + if table_type != TableType::Table { + return Err(DataFusionError::NotImplemented(format!( + "CREATE TABLE AS SELECT only supports native Paimon tables, got '{table_type}'" + ))); + } + + let df = + Box::pin(self.plan_query(state, Statement::Query(query), options, time_travel_sources)) + .await?; + let (state, source) = df.into_parts(); + let task_ctx = state.task_ctx(); + // Optimization can narrow nullability beyond the query's declared schema. + let analyzed = + state + .analyzer() + .execute_and_check(source, state.config_options(), |_, _| {})?; + let schema = create_table_schema( + ct, + partition_keys, + enable_ident_normalization, + Some(analyzed.schema().as_arrow()), + table_options, + )?; + if schema.fields().is_empty() { + return Err(DataFusionError::Plan( + "CREATE TABLE AS SELECT requires at least one output column".into(), + )); + } + for field in schema.fields() { + validate_time_precision(field.data_type())?; + } + let write_schema = paimon::arrow::build_target_arrow_schema(schema.fields()) + .map_err(to_datafusion_error)?; + let optimized = state.optimizer().optimize(analyzed, &state, |_, _| {})?; + let plan = state + .query_planner() + .create_physical_plan(&optimized, &state) + .await?; + + match catalog.create_table_exclusive(&identifier, schema).await { + Ok(()) => {} + Err(paimon::Error::TableAlreadyExist { .. }) if ct.if_not_exists => { Review Comment: **[P1] Preserve an uncertain create outcome on S3 retries** If the initial conditional PUT of `schema-0` succeeds and the response is lost, OpenDAL's retry can receive HTTP 412. The filesystem catalog maps that response to `TableAlreadyExist`, and this branch returns success for `CREATE TABLE IF NOT EXISTS ... AS SELECT` before populating the newly created table. Re-running the statement also skips the empty table. Please propagate an unknown create outcome and reserve this success path for a definite creation conflict. This finding is based on static analysis of the filesystem catalog and the OpenDAL 0.59.5 retry path. ########## crates/integrations/datafusion/src/relation_planner.rs: ########## @@ -39,6 +39,31 @@ use crate::catalog::{ }; use crate::table::{ObjectTableProvider, PaimonTableProvider}; +/// Resolves statement-local TABLE bindings to snapshot scans with the original table qualifier. +#[derive(Debug)] +pub(crate) struct SnapshotRelationPlanner( + pub HashMap<String, (TableReference, Arc<dyn TableProvider>)>, +); + +impl RelationPlanner for SnapshotRelationPlanner { + fn plan_relation( + &self, + relation: TableFactor, + _: &mut dyn RelationPlannerContext, + ) -> DFResult<RelationPlanning> { + if let TableFactor::TableFunction { + expr: ast::Expr::Identifier(binding), + alias, + } = &relation + { + if let Some((table, provider)) = self.0.get(&binding.value) { + return plan_scan(alias.clone(), table.clone(), provider.clone()); Review Comment: **[P2] Keep distinct qualifiers for unaliased scans of different snapshots** `SELECT COUNT(*) FROM events VERSION AS OF 1 JOIN events VERSION AS OF 2 USING (id)` returned the expected count on the base. It now fails with a duplicate qualified field `events.id` because both snapshot scans inherit `events` as their qualifier. Please assign distinct qualifiers when snapshot scans in the same join would collide and preserve any explicit aliases. ########## crates/integrations/datafusion/src/sql_context.rs: ########## @@ -936,137 +957,76 @@ impl SQLContext { )) .await } - Statement::Query(_) | Statement::Explain { .. } => { - let mut statement = statements[0].clone(); - if let Statement::Explain { - statement: inner, .. - } = &mut statement - { - if let Statement::Merge(merge) = inner.as_mut() { - *merge = self.normalize_merge_into(merge).await?; - } - } - let current_catalog = self.current_catalog_name(); - let current_database = self - .ctx - .state() - .config_options() - .catalog - .default_schema - .clone(); - let expanded = crate::sql_function::expand_statement( - &self.ctx.state(), - statement, - &self.catalogs, - ¤t_catalog, - ¤t_database, - ) - .await?; - crate::read_files::sql(&self.ctx, &expanded.to_string(), options).await - } _ => crate::read_files::sql(&self.ctx, sql, options).await, } } - /// Rewrite time-travel references to temporary snapshot providers. - /// The tracker removes registrations after planning. - async fn handle_time_travel_query( + fn rewrite_expressions(statement: &mut Statement, state: &SessionState) -> DFResult<bool> { + let binary = crate::binary_cast::rewrite(statement)?; + let regex = crate::regex_operators::rewrite(statement); + let lateral = crate::lateral_views::rewrite(statement)?; + let compat = crate::expression_compat::rewrite(statement, state)?; + Ok(binary || regex || lateral || compat) + } + + async fn plan_query( &self, - sql: &str, - sql_options: SQLOptions, + mut state: SessionState, + mut statement: Statement, + options: SQLOptions, + sources: &[TimeTravelSource], ) -> DFResult<DataFrame> { - let mut tracker = crate::merge_into::TempTableTracker::new(self); - - let version_clauses = extract_all_version_as_of(sql); - let timestamp_clauses = extract_all_timestamp_as_of(sql); - - if version_clauses.is_empty() && timestamp_clauses.is_empty() { - return Err(DataFusionError::Plan( - "Failed to parse time-travel clause in SQL".to_string(), - )); - } - - let mut replacements: Vec<((usize, usize), String)> = Vec::new(); - - for info in &version_clauses { - let table_ref: datafusion::common::TableReference = info.table_name.as_str().into(); - let (catalog, catalog_name, identifier) = - self.resolve_table_name_from_ref(&table_ref)?; - let provider = self - .time_travel_provider( - &catalog, - &catalog_name, - &identifier, - TimeTravelClause::Version(info.version.clone()), - ) - .await?; - - let uuid_name = format!("__paimon_tt_{}", uuid::Uuid::new_v4().as_simple()); - self.register_temp_table(uuid_name.as_str(), provider)?; - tracker.register(&uuid_name); - replacements.push((info.clause_range, uuid_name)); + Self::rewrite_expressions(&mut statement, &state)?; + if let Statement::Explain { + statement: inner, .. + } = &mut statement + { + if let Statement::Merge(merge) = inner.as_mut() { + *merge = self.normalize_merge_into(merge).await?; + } } - - for info in ×tamp_clauses { - let table_ref: datafusion::common::TableReference = info.table_name.as_str().into(); + let session_catalog = &state.config_options().catalog; + let mut bindings = HashMap::new(); + for source in sources { + let table = object_name_to_table_reference( + source.table.clone(), + state.config_options().sql_parser.enable_ident_normalization, + )?; let (catalog, catalog_name, identifier) = - self.resolve_table_name_from_ref(&table_ref)?; - let millis = Self::parse_timestamp_to_millis(&info.timestamp)?; + self.resolve_table_reference(&table, session_catalog)?; let provider = self - .time_travel_provider( - &catalog, - &catalog_name, - &identifier, - TimeTravelClause::TimestampMillis(millis), - ) + .time_travel_provider(&catalog, &catalog_name, &identifier, &source.clause) .await?; - - let uuid_name = format!("__paimon_tt_{}", uuid::Uuid::new_v4().as_simple()); - self.register_temp_table(uuid_name.as_str(), provider)?; - tracker.register(&uuid_name); - replacements.push((info.clause_range, uuid_name)); - } - - // Replace from right to left to preserve byte offsets. - replacements.sort_by_key(|r| std::cmp::Reverse(r.0 .0)); - - let mut rewritten_sql = sql.to_string(); - for ((start, end), uuid_name) in &replacements { - rewritten_sql = format!( - "{}{}{}", - &rewritten_sql[..*start], - uuid_name, - &rewritten_sql[*end..] - ); + bindings.insert(source.binding.clone(), (table, provider)); } - - let current_catalog = self.current_catalog_name(); - let current_database = self - .ctx - .state() - .config_options() - .catalog - .default_schema - .clone(); - let expanded = crate::sql_function::expand_sql( - &self.ctx.state(), - &rewritten_sql, + let expanded = crate::sql_function::expand_statement( + &state, + statement, &self.catalogs, - ¤t_catalog, - ¤t_database, + &session_catalog.default_catalog, + &session_catalog.default_schema, ) .await?; - crate::read_files::sql(&self.ctx, &expanded, sql_options).await + let sql = expanded.to_string(); + if !bindings.is_empty() { + state.register_relation_planner(Arc::new( + crate::relation_planner::SnapshotRelationPlanner(bindings), + ))?; + } + // USE can change the shared session while catalog lookups are in flight. + let context = SessionContext::new_with_state(state); Review Comment: **[P2] Register time-travel prepared statements in the original session** `PREPARE historical AS SELECT id FROM events VERSION AS OF 1` succeeds on this revision. Running `EXECUTE historical` in the same `SQLContext` then fails with `Prepared statement 'historical' does not exist`. DataFusion stores the prepared plan on the temporary context created here. The same sequence returns the historical row on the base. Please plan against the captured session state and register the resulting prepared plan on the original context. ########## crates/integrations/datafusion/src/sql_context.rs: ########## @@ -3322,284 +3270,144 @@ fn validate_persistent_create_view(create_view: &CreateView) -> DFResult<()> { Ok(()) } -/// Quick check whether the SQL looks like a CREATE TABLE statement. -/// Skips leading whitespace, `--` line comments, and `/* */` block comments. -fn looks_like_create_table(sql: &str) -> bool { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - // Skip leading whitespace and comments - loop { - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - if i + 1 < len && bytes[i] == b'-' && bytes[i + 1] == b'-' { - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } - continue; - } - if i + 1 < len && bytes[i] == b'/' && bytes[i + 1] == b'*' { - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } - continue; - } - break; - } - // Match "CREATE" then whitespace then optional "TEMPORARY"/"TEMP" then "TABLE" (all ASCII, byte-safe) - if i + 6 > len || !bytes[i..i + 6].eq_ignore_ascii_case(b"CREATE") { - return false; - } - i += 6; - if i >= len || !bytes[i].is_ascii_whitespace() { - return false; - } - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - // Skip optional TEMPORARY or TEMP keyword - if i + 9 <= len && bytes[i..i + 9].eq_ignore_ascii_case(b"TEMPORARY") { - i += 9; - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - } else if i + 4 <= len && bytes[i..i + 4].eq_ignore_ascii_case(b"TEMP") { - i += 4; - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - } - // After optional TEMPORARY/TEMP, reject CREATE TEMPORARY VIEW / CREATE TEMP VIEW - if i + 4 <= len && bytes[i..i + 4].eq_ignore_ascii_case(b"VIEW") { - return false; - } - i + 5 <= len && bytes[i..i + 5].eq_ignore_ascii_case(b"TABLE") -} - -/// Find `PARTITIONED BY` keyword position, skipping string literals and comments. -fn find_partitioned_by(sql: &str) -> Option<(usize, usize)> { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - while i < len { - match bytes[i] { - b'\'' => { - i += 1; - while i < len { - if bytes[i] == b'\'' { - i += 1; - if i < len && bytes[i] == b'\'' { - i += 1; - } else { - break; - } - } else { - i += 1; - } - } - } - b'-' if i + 1 < len && bytes[i + 1] == b'-' => { - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } - } - b'/' if i + 1 < len && bytes[i + 1] == b'*' => { - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } - } - b if b.is_ascii_alphabetic() && i + 11 <= len => { - if bytes[i..i + 11].eq_ignore_ascii_case(b"PARTITIONED") { - let rest = &bytes[i + 11..]; - let ws = rest.iter().take_while(|b| b.is_ascii_whitespace()).count(); - if ws > 0 - && i + 11 + ws + 2 <= len - && rest[ws..ws + 2].eq_ignore_ascii_case(b"BY") - { - let by_end = i + 11 + ws + 2; - return Some((i, by_end)); - } - } - i += 1; - } - _ => { - i += 1; - } +fn create_table_parser(sql: &str) -> Option<Parser<'_>> { + if !matches!( + sql.trim_start().as_bytes().first(), + Some(b'c' | b'C' | b'-' | b'/' | b';') + ) { + return None; + } + let mut parser = Parser::new(&GenericDialect).try_with_sql(sql).ok()?; + while parser.consume_token(&Token::SemiColon) {} + if !parser.parse_keyword(Keyword::CREATE) { + return None; + } + while let Token::Word(word) = parser.next_token().token { + match word.keyword { + Keyword::TABLE => return Some(parser), + Keyword::OR + | Keyword::REPLACE + | Keyword::ALTER + | Keyword::TEMP + | Keyword::TEMPORARY + | Keyword::EXTERNAL + | Keyword::GLOBAL + | Keyword::LOCAL + | Keyword::TRANSIENT + | Keyword::VOLATILE + | Keyword::DYNAMIC + | Keyword::ICEBERG + | Keyword::SNAPSHOT => {} + _ => break, } } None } -/// Parse a single partition column token, handling quoted identifiers. -fn parse_partition_column(token: &str, enable_ident_normalization: bool) -> DFResult<String> { - let trimmed = token.trim(); - if trimmed.is_empty() { - return Err(DataFusionError::Plan( - "Empty column name in PARTITIONED BY".to_string(), - )); - } - - let first = trimmed.as_bytes()[0]; - if first == b'"' || first == b'`' { - let mut value = String::new(); - let mut end = None; - let mut chars = trimmed[1..].char_indices().peekable(); - while let Some((idx, ch)) = chars.next() { - if ch == first as char { - if chars.peek().is_some_and(|(_, next)| *next == first as char) { - value.push(ch); - chars.next(); - } else { - end = Some(1 + idx + ch.len_utf8()); - break; +fn validate_ctas_option_clauses(sql: &str) -> DFResult<()> { + let Some(mut parser) = create_table_parser(sql) else { + return Ok(()); + }; + let _ = parser.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); + parser.parse_object_name(false)?; + let mut depth = 0; + let mut seen_options = false; + loop { + match parser.next_token().token { + Token::EOF => return Ok(()), + Token::LParen => depth += 1, + Token::RParen => depth -= 1, + Token::Word(word) if depth == 0 => match word.keyword { + Keyword::AS => return Ok(()), + Keyword::WITH | Keyword::TBLPROPERTIES | Keyword::OPTIONS + if parser.peek_token_ref().token == Token::LParen => + { + if seen_options { + return Err(plan_datafusion_err!( + "CREATE TABLE AS SELECT accepts only one table options clause" + )); + } + seen_options = true; } - } else { - value.push(ch); - } - } - if let Some(end) = end { - if trimmed[end..].trim().is_empty() { - return Ok(normalize_schema_identifier( - &Ident::with_quote(first as char, value), - enable_ident_normalization, - )); - } + _ => {} + }, + _ => {} } - return Err(DataFusionError::Plan(format!( - "Invalid quoted identifier in PARTITIONED BY: {trimmed}" - ))); - } - - let parts: Vec<&str> = trimmed.split_whitespace().collect(); - match parts.len() { - 1 => Ok(normalize_schema_identifier( - &Ident::new(parts[0]), - enable_ident_normalization, - )), - _ => Err(DataFusionError::Plan(format!( - "PARTITIONED BY column '{}' should not specify a type. \ - Use column references only, e.g. PARTITIONED BY ({})", - parts[0], parts[0] - ))), } } -fn split_partition_columns(inner: &str) -> DFResult<Vec<&str>> { - let mut columns = Vec::new(); - let mut start = 0; - let mut quote = None; - let mut chars = inner.char_indices().peekable(); - while let Some((idx, ch)) = chars.next() { - match quote { - Some(q) if ch == q => { - if chars.peek().is_some_and(|(_, next)| *next == q) { - chars.next(); - } else { - quote = None; - } - } - Some(_) => {} - None if ch == '"' || ch == '`' => quote = Some(ch), - None if ch == ',' => { - columns.push(&inner[start..idx]); - start = idx + ch.len_utf8(); - } - None => {} +fn sql_byte_offset(sql: &str, location: Location) -> usize { + let mut offset = 0; + for (index, line) in sql.split_inclusive('\n').enumerate() { + if index + 1 == location.line as usize { + return offset + + line + .char_indices() + .nth(location.column as usize - 1) + .map_or(line.len(), |(index, _)| index); } + offset += line.len(); } - if quote.is_some() { - return Err(DataFusionError::Plan( - "Unterminated quoted identifier in PARTITIONED BY".to_string(), - )); - } - columns.push(&inner[start..]); - Ok(columns) + offset } -/// Extract `PARTITIONED BY (col1, col2, ...)` from SQL before parsing. -/// -/// Paimon only allows column references (no types) in PARTITIONED BY. -/// Since sqlparser's GenericDialect requires types in column definitions, -/// we extract and validate the clause ourselves, then strip it from the SQL -/// so sqlparser can parse the rest. +/// GenericDialect expects typed Hive partition columns. Parse Paimon's list of +/// column references separately and leave the AS query intact. fn extract_partition_by( sql: &str, enable_ident_normalization: bool, ) -> DFResult<(String, Vec<String>)> { - let Some((kw_start, by_end)) = find_partitioned_by(sql) else { - return Ok((sql.to_string(), vec![])); + let Some(mut parser) = create_table_parser(sql) else { + return Ok((sql.to_owned(), vec![])); }; - - let after_by = sql[by_end..].trim_start(); - let paren_start = by_end + (sql[by_end..].len() - after_by.len()); - - if !after_by.starts_with('(') { - return Err(DataFusionError::Plan( - "Expected '(' after PARTITIONED BY".to_string(), - )); - } - - let inner_start = paren_start + 1; - let mut depth = 1; - let mut paren_end = None; - let mut quote = None; - let mut chars = sql[inner_start..].char_indices().peekable(); - while let Some((i, ch)) = chars.next() { - match quote { - Some(q) if ch == q => { - if chars.peek().is_some_and(|(_, next)| *next == q) { - chars.next(); - } else { - quote = None; - } + let mut depth = 0; + let start = loop { + let token = parser.next_token(); + match token.token { + Token::LParen => depth += 1, + Token::RParen => depth -= 1, + Token::Word(word) if depth == 0 && word.keyword == Keyword::AS => { Review Comment: **[P2] Preserve partition keys when the target name contains `AS`** `CREATE TABLE default.as (id INT) PARTITIONED BY (id)` succeeds with no partition keys on this revision. The scanner treats the `as` component of the table name as the query separator. The SQL parser still accepts the Hive partition clause, which ordinary Paimon table creation ignores. Please consume the optional `IF NOT EXISTS` and the complete target name before scanning for `PARTITIONED BY` and the query separator. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
