leaves12138 commented on code in PR #1438:
URL: https://github.com/apache/paimon-rust/pull/1438#discussion_r4598468140


##########
crates/integrations/datafusion/src/read_files/projection.rs:
##########
@@ -0,0 +1,350 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::ops::ControlFlow;
+use std::sync::{Arc, Mutex};
+
+use datafusion::catalog::{TableFunctionArgs, TableFunctionImpl};
+use datafusion::common::ExprSchema;
+use datafusion::dataframe::DataFrame;
+use datafusion::datasource::TableProvider;
+use datafusion::error::{DataFusionError, Result};
+use datafusion::execution::context::SQLOptions;
+use datafusion::execution::session_state::SessionState;
+use datafusion::logical_expr::{Expr, LogicalPlan};
+use datafusion::prelude::SessionContext;
+use datafusion::sql::sqlparser::ast::{
+    Cte, Query, Select, SelectItem, TableFactor, VisitMut, VisitorMut,
+};
+use datafusion::sql::sqlparser::{dialect::GenericDialect, parser::Parser};
+use datafusion::sql::unparser::Unparser;
+
+use super::{error, is_read_files, GENERATED_METADATA};
+
+/// The SQL planner has no hidden-column facility. Expand wildcards against the
+/// FROM schema before planning the real projection, excluding only generated
+/// metadata on direct read_files sources. Explicit exports through a subquery,
+/// CTE or view are ordinary visible columns.
+pub(crate) fn sql<'a>(
+    ctx: &'a SessionContext,
+    sql: &'a str,
+    options: SQLOptions,
+) -> futures::future::BoxFuture<'a, Result<DataFrame>> {
+    // Keep this additional planning layer off SQLContext's already large 
future.
+    Box::pin(async move {
+        // The AST check below is authoritative; this avoids reparsing 
ordinary SQL.
+        if !sql.to_ascii_lowercase().contains("read_files") {
+            return ctx.sql_with_options(sql, options).await;
+        }
+        let mut statements = Parser::parse_sql(&GenericDialect, sql)?;
+        let [statement] = statements.as_mut_slice() else {
+            return ctx.sql_with_options(sql, options).await;
+        };
+        struct FindSource;
+        impl VisitorMut for FindSource {
+            type Break = ();
+            fn pre_visit_table_factor(&mut self, factor: &mut TableFactor) -> 
ControlFlow<()> {
+                let name = match factor {
+                    TableFactor::Table {
+                        name,
+                        args: Some(_),
+                        ..
+                    }
+                    | TableFactor::Function { name, .. } => name,
+                    _ => return ControlFlow::Continue(()),
+                };
+                if is_read_files(name) {
+                    ControlFlow::Break(())
+                } else {
+                    ControlFlow::Continue(())
+                }
+            }
+        }
+        if statement.visit(&mut FindSource).is_continue() {
+            return ctx.sql_with_options(sql, options).await;
+        }
+        let mut state = ctx.state();
+        let Some(function) = 
state.table_functions().get("read_files").cloned() else {
+            return ctx.sql_with_options(sql, options).await;
+        };
+        // Bind each source once for both schema planning and execution. The 
cache
+        // belongs to this statement's cloned state, never to the shared 
session.
+        state.register_udtf(
+            "read_files",
+            Arc::new(CachedSource {
+                function: function.function().clone(),
+                providers: Mutex::new(HashMap::new()),
+            }),
+        );
+        let mut rewriter = Wildcards {
+            state: state.clone(),
+            scopes: Vec::new(),
+            originals: Vec::new(),
+            rewritten: HashMap::new(),
+        };
+        if let ControlFlow::Break(error) = statement.visit(&mut rewriter) {
+            return Err(*error);
+        }
+        SessionContext::new_with_state(state)
+            .sql_with_options(&statement.to_string(), options)
+            .await
+    })
+}
+
+#[derive(Debug)]
+struct CachedSource {
+    function: Arc<dyn TableFunctionImpl>,
+    providers: Mutex<HashMap<String, Arc<dyn TableProvider>>>,
+}
+
+impl TableFunctionImpl for CachedSource {
+    fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn 
TableProvider>> {
+        let key = format!("{:?}", args.exprs());
+        let mut providers = self
+            .providers
+            .lock()
+            .map_err(|_| error("source cache poisoned"))?;
+        if let Some(provider) = providers.get(&key) {
+            return Ok(provider.clone());
+        }
+        let provider = self.function.call_with_args(args)?;
+        providers.insert(key, provider.clone());
+        Ok(provider)
+    }
+}
+
+struct Wildcards {
+    state: SessionState,
+    scopes: Vec<Vec<ScopedCte>>,
+    originals: Vec<usize>,
+    rewritten: HashMap<usize, Query>,
+}
+
+#[derive(Clone)]
+struct ScopedCte {
+    cte: Cte,
+    // Identity of the original boxed AST node during this visit, never 
dereferenced.
+    identity: usize,
+    recursive: bool,
+}
+
+fn direct_sources(factor: &TableFactor, state: &SessionState, sources: &mut 
Vec<String>) {
+    match factor {
+        TableFactor::Table {
+            name,
+            args: Some(_),
+            alias,
+            ..
+        }
+        | TableFactor::Function { name, alias, .. }
+            if is_read_files(name) =>
+        {
+            sources.push(alias.as_ref().map_or_else(
+                || "read_files()".to_string(),
+                |alias| {
+                    if alias.name.quote_style.is_none()
+                        && 
state.config_options().sql_parser.enable_ident_normalization
+                    {
+                        alias.name.value.to_ascii_lowercase()
+                    } else {
+                        alias.name.value.clone()
+                    }
+                },
+            ));
+        }
+        TableFactor::NestedJoin {
+            table_with_joins, ..
+        } => {
+            direct_sources(&table_with_joins.relation, state, sources);
+            for join in &table_with_joins.joins {
+                direct_sources(&join.relation, state, sources);
+            }
+        }
+        _ => {}
+    }
+}
+
+fn aliased_source_join(factor: &TableFactor, state: &SessionState) -> bool {
+    let TableFactor::NestedJoin {
+        table_with_joins,
+        alias,
+    } = factor
+    else {
+        return false;
+    };
+    let mut sources = Vec::new();
+    direct_sources(factor, state, &mut sources);
+    (!sources.is_empty() && alias.is_some())
+        || aliased_source_join(&table_with_joins.relation, state)
+        || table_with_joins
+            .joins
+            .iter()
+            .any(|join| aliased_source_join(&join.relation, state))
+}
+
+impl Wildcards {
+    fn expand(&self, select: &mut Select) -> Result<()> {
+        if !select.projection.iter().any(|item| {
+            matches!(
+                item,
+                SelectItem::Wildcard(_) | SelectItem::QualifiedWildcard(_, _)
+            )
+        }) {
+            return Ok(());
+        }
+        let mut sources = Vec::new();
+        for from in &select.from {
+            direct_sources(&from.relation, &self.state, &mut sources);
+            for join in &from.joins {
+                direct_sources(&join.relation, &self.state, &mut sources);
+            }
+        }
+        if sources.is_empty() {
+            return Ok(());
+        }
+        // DataFusion collapses all qualifiers under a parenthesized join alias
+        // before projecting. At that point generated and explicitly exported
+        // metadata cannot be distinguished reliably. Reject this unsupported
+        // wildcard shape instead of silently exposing or dropping a column.
+        if select.from.iter().any(|from| {
+            aliased_source_join(&from.relation, &self.state)
+                || from
+                    .joins
+                    .iter()
+                    .any(|join| aliased_source_join(&join.relation, 
&self.state))
+        }) {
+            return Err(error("wildcards on aliased parenthesized joins 
containing read_files are not supported; alias the individual sources 
instead"));
+        }
+        let ctes = self
+            .scopes
+            .last()
+            .unwrap()
+            .iter()
+            .map(|scoped| {
+                let mut cte = scoped.cte.clone();
+                if let Some(query) = self.rewritten.get(&scoped.identity) {
+                    *cte.query = query.clone();
+                }
+                cte.to_string()
+            })
+            .collect::<Vec<_>>();
+        let prefix = if ctes.is_empty() {
+            String::new()
+        } else {
+            let recursive = if self.scopes.last().unwrap().iter().any(|cte| 
cte.recursive) {
+                "RECURSIVE "
+            } else {
+                ""
+            };
+            format!("WITH {recursive}{} ", ctes.join(", "))
+        };
+        let from = select
+            .from
+            .iter()
+            .map(ToString::to_string)
+            .collect::<Vec<_>>()
+            .join(", ");
+        let mut projection = Vec::new();
+        for item in &select.projection {
+            if !matches!(
+                item,
+                SelectItem::Wildcard(_) | SelectItem::QualifiedWildcard(_, _)
+            ) {
+                projection.push(item.clone());
+                continue;
+            }
+            let sql = format!("{prefix}SELECT {item} FROM {from}");
+            let state = self.state.clone();
+            let plan = crate::runtime::block_on_with_runtime(
+                async move { state.create_logical_plan(&sql).await },
+                "read_files: wildcard planning thread panicked",
+            )?;
+            let LogicalPlan::Projection(plan) = plan else {
+                return Err(error("expected a wildcard projection"));
+            };
+            for expr in plan.expr {
+                if let Expr::Column(column) = &expr {
+                    let field = plan.input.schema().field_from_column(column)?;
+                    if field.metadata().contains_key(GENERATED_METADATA)
+                        && column
+                            .relation
+                            .as_ref()
+                            .is_some_and(|name| sources.iter().any(|source| 
source == name.table()))
+                    {
+                        continue;
+                    }
+                }
+                projection.push(SelectItem::UnnamedExpr(
+                    Unparser::default().expr_to_sql(&expr)?,
+                ));

Review Comment:
   [P2] Preserve aliases when expanding wildcard replacement expressions
   
   Wildcard `REPLACE` produces an `Expr::Alias`, but `Unparser::expr_to_sql` 
strips that alias; wrapping the resulting expression in 
`SelectItem::UnnamedExpr` therefore loses the requested output column name. For 
a TEXT file containing `hello\n`, `SELECT * REPLACE (upper(value) AS value) 
FROM read_files('memory://review/data.txt')` now names its data column 
`upper(read_files().value)` instead of `value`. Consequently, `SELECT q.value 
FROM (SELECT * REPLACE (upper(value) AS value) FROM 
read_files('memory://review/data.txt')) q` fails with `column 'value' not found 
in 'q'`, whereas native DataFusion returns `HELLO`. This also changes the 
schema inferred by CTAS/import consumers. Please preserve planned aliases as 
aliased select items rather than discarding them during wildcard expansion, and 
add a regression for a replacement expression referenced by an outer query.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to