SteNicholas commented on code in PR #1315:
URL: https://github.com/apache/paimon-rust/pull/1315#discussion_r4453213526


##########
crates/integrations/datafusion/src/merge_into/native.rs:
##########
@@ -0,0 +1,715 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Storage execution for DataFusion's native MERGE logical plan.
+
+use std::collections::HashSet;
+use std::fmt;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use datafusion::arrow::array::{Array, ArrayRef, BooleanArray, Int32Array};
+use datafusion::arrow::compute::{cast, filter_record_batch};
+use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions};
+use datafusion::catalog::Session;
+use datafusion::common::tree_node::{Transformed, TreeNode};
+use datafusion::common::{Column, DFSchema, DFSchemaRef};
+use datafusion::datasource::sink::{DataSink, DataSinkExec};
+use datafusion::datasource::MemTable;
+use datafusion::error::{DataFusionError, Result};
+use datafusion::execution::{SendableRecordBatchStream, TaskContext};
+use datafusion::logical_expr::dml::{MergeIntoAction, MergeIntoClause, 
MergeIntoClauseKind};
+use datafusion::logical_expr::{col, expr::Case, lit, Expr, JoinType, Operator};
+use datafusion::physical_plan::{DisplayAs, ExecutionPlan};
+use datafusion::prelude::{DataFrame, SessionContext};
+use futures::TryStreamExt;
+use paimon::spec::CoreOptions;
+use paimon::table::{CopyOnWriteMergeWriter, DataEvolutionDeleteWriter, Table};
+
+use super::{
+    build_partition_set_from_batches, extract_tracking_columns, 
is_delete_conflict,
+    is_row_id_conflict, read_cow_target_batches, retry_on_conflict, 
validate_update_columns,
+};
+use crate::error::to_datafusion_error;
+use crate::table::PaimonTableProvider;
+
+const PRESENT: &str = "__paimon_merge_present";
+const FILE: &str = "__paimon_file_idx";
+const OFFSET: &str = "__paimon_row_offset";
+
+fn column_name(index: usize, target_len: usize) -> String {
+    if index < target_len {
+        format!("__paimon_merge_t_{index}")
+    } else {
+        format!("__paimon_merge_s_{}", index - target_len)
+    }
+}
+
+/// Bind columns by their position in the native target+source schema. SQL 
aliases
+/// and quoted names have already been resolved by DataFusion's SQL planner.
+fn bind_expr(expr: Expr, schema: &DFSchema, target_len: usize) -> Result<Expr> 
{
+    expr.transform(|expr| match expr {
+        Expr::Column(column) => Ok(Transformed::yes(col(column_name(
+            schema.index_of_column(&column)?,
+            target_len,
+        )))),
+        Expr::ScalarSubquery(_)
+        | Expr::Exists(_)
+        | Expr::InSubquery(_)
+        | Expr::OuterReferenceColumn(_, _) => 
Err(DataFusionError::NotImplemented(
+            "Paimon MERGE expressions containing subqueries are not 
supported".into(),
+        )),
+        other => Ok(Transformed::no(other)),
+    })
+    .map(|transformed| transformed.data)
+}
+
+pub(crate) fn plan_merge(
+    table: Table,
+    source: Arc<dyn ExecutionPlan>,
+    merge_schema: DFSchemaRef,
+    on: Expr,
+    mut clauses: Vec<MergeIntoClause>,
+) -> Result<Arc<dyn ExecutionPlan>> {
+    if table.is_branch_reference() {
+        return Err(DataFusionError::Plan(format!(
+            "MERGE INTO on Paimon branch '{}' is not supported",
+            table.branch()
+        )));
+    }
+    if table.is_time_traveled() {
+        return Err(DataFusionError::Plan(
+            "Cannot execute MERGE INTO on a time-travel table".into(),
+        ));
+    }
+    let de = 
CoreOptions::new(table.schema().options()).data_evolution_enabled();
+    if !table.schema().trimmed_primary_keys().is_empty() {
+        return Err(DataFusionError::Plan(
+            "MERGE INTO on primary-key tables without data-evolution is not 
supported".into(),
+        ));
+    }
+    let target_len = merge_schema.fields().len() - 
source.schema().fields().len();
+    let mut update_columns = Vec::new();
+    let mut matched_count = 0;
+    for clause in &mut clauses {
+        match clause.kind {
+            MergeIntoClauseKind::NotMatchedBySource => {
+                return Err(DataFusionError::NotImplemented(format!(
+                    "WHEN NOT MATCHED BY SOURCE is not yet supported for {} 
MERGE INTO",
+                    if de { "data evolution" } else { "CoW" }
+                )));
+            }
+            MergeIntoClauseKind::Matched => {
+                if de && matched_count > 0 {
+                    return Err(DataFusionError::NotImplemented(
+                        "Multiple WHEN MATCHED clauses are not yet 
supported".into(),
+                    ));
+                }
+                if de && clause.predicate.is_some() {
+                    return Err(DataFusionError::NotImplemented(
+                        "WHEN MATCHED AND <predicate> is not yet 
supported".into(),
+                    ));
+                }
+                matched_count += 1;
+            }
+            _ => {}
+        }
+        if let Some(predicate) = clause.predicate.take() {
+            clause.predicate = Some(bind_expr(predicate, &merge_schema, 
target_len)?);
+        }
+        match &mut clause.action {
+            MergeIntoAction::Update(assignments) => {
+                let columns = assignments
+                    .iter()
+                    .map(|(c, _)| c.clone())
+                    .collect::<Vec<_>>();
+                validate_update_columns(&columns, table.schema().fields())?;
+                if de {
+                    // Validate storage constraints without scanning or 
writing.
+                    let _ = table
+                        .new_write_builder()
+                        .new_data_evolution_writer(columns.clone())
+                        .map_err(to_datafusion_error)?;
+                }
+                for (name, value) in assignments {
+                    if !update_columns.contains(name) {
+                        update_columns.push(name.clone());
+                    }
+                    *value = bind_expr(value.clone(), &merge_schema, 
target_len)?;
+                }
+            }
+            MergeIntoAction::Insert { columns, values } => {
+                if columns.is_empty() {
+                    *columns = table
+                        .schema()
+                        .fields()
+                        .iter()
+                        .map(|f| f.name().to_string())
+                        .collect();
+                }
+                validate_update_columns(columns, table.schema().fields())?;
+                for value in values {
+                    *value = bind_expr(value.clone(), &merge_schema, 
target_len)?;
+                }
+            }
+            MergeIntoAction::Delete if de => {
+                let _ = table
+                    .new_write_builder()
+                    .new_delete()
+                    .map_err(to_datafusion_error)?;
+            }
+            MergeIntoAction::Delete => {}
+        }
+    }
+    let on = bind_expr(on, &merge_schema, target_len)?;
+    let sink = PaimonMergeSink {
+        table,
+        schema: source.schema(),
+        merge_schema,
+        target_len,
+        on,
+        clauses,
+        update_columns,
+        de,
+    };
+    Ok(Arc::new(DataSinkExec::new(source, Arc::new(sink), None)))
+}
+
+#[derive(Debug)]
+struct PaimonMergeSink {
+    table: Table,
+    schema: SchemaRef,
+    merge_schema: DFSchemaRef,
+    target_len: usize,
+    on: Expr,
+    clauses: Vec<MergeIntoClause>,
+    update_columns: Vec<String>,
+    de: bool,
+}
+
+impl DisplayAs for PaimonMergeSink {
+    fn fmt_as(
+        &self,
+        _: datafusion::physical_plan::DisplayFormatType,
+        f: &mut fmt::Formatter,
+    ) -> fmt::Result {
+        write!(f, "PaimonMerge: table={}", self.table.identifier())
+    }
+}
+
+#[async_trait]
+impl DataSink for PaimonMergeSink {
+    fn schema(&self) -> &SchemaRef {
+        &self.schema
+    }
+
+    async fn write_all(
+        &self,
+        data: SendableRecordBatchStream,
+        context: &Arc<TaskContext>,
+    ) -> Result<u64> {
+        // Capture the source once, including self-merges and volatile 
expressions.
+        // Each conflict retry rebuilds the target scan and writers from fresh 
metadata.
+        let source: Vec<RecordBatch> = data.try_collect().await?;
+        retry_on_conflict(
+            if self.de {
+                "MERGE INTO"
+            } else {
+                "CoW MERGE INTO"
+            },
+            if self.de {
+                is_row_id_conflict
+            } else {
+                is_delete_conflict
+            },
+            || self.execute_once(&source, context),
+        )
+        .await
+    }
+}
+
+impl PaimonMergeSink {
+    async fn execute_once(
+        &self,
+        source: &[RecordBatch],
+        context: &Arc<TaskContext>,
+    ) -> Result<u64> {
+        let ctx = SessionContext::new_with_config_rt(
+            context.session_config().clone(),
+            context.runtime_env(),
+        );

Review Comment:
   Creating a new SessionContext from only SessionConfig and RuntimeEnv drops 
the original session's variable providers, which are stored in ExecutionProps. 
For example, after registering a user-defined variable provider on the caller's 
SessionContext, `WHEN MATCHED THEN UPDATE SET value = @merge_value` can be 
planned there, but evaluating the assignment through this new context fails 
because the provider is absent. DataFusion's constant evaluator explicitly 
leaves ScalarVariable expressions unevaluated, so optimization does not remove 
this dependency. The previous implementation evaluated these queries through 
the caller's context. Please preserve the relevant session execution state when 
rebuilding the target plan, and add coverage for variables in ON, matched 
predicates, and UPDATE assignments.



##########
crates/integrations/datafusion/src/sql_context.rs:
##########
@@ -1814,14 +1821,71 @@ impl SQLContext {
             }
         };
         Self::ensure_main_branch_write_target(&table_name, "MERGE INTO")?;
-        let (catalog, _catalog_name, identifier) = 
self.resolve_catalog_and_table(&table_name)?;
-
-        let table = catalog
-            .get_table(&identifier)
-            .await
-            .map_err(to_datafusion_error)?;
-
-        crate::merge_into::execute_merge_into(self, merge, table, 
enable_ident_normalization).await
+        let mut merge = merge.clone();
+        // Existing not-matched expressions resolve unqualified columns against
+        // the source. DataFusion 55 resolves them against the combined schema,
+        // which can make common column names ambiguous. Preserve source scope
+        // while leaving name/type validation to the native planner.
+        let source_qualifier = match &merge.source {
+            TableFactor::Table {
+                alias: Some(alias), ..
+            }
+            | TableFactor::Derived {
+                alias: Some(alias), ..
+            } => vec![alias.name.clone()],
+            TableFactor::Table { name, .. } => name
+                .0
+                .iter()
+                .filter_map(|p| p.as_ident().cloned())
+                .collect(),
+            _ => vec![],
+        };
+        if !source_qualifier.is_empty() {
+            for clause in &mut merge.clauses {
+                if let 
datafusion::sql::sqlparser::ast::MergeAction::Insert(insert) =
+                    &mut clause.action
+                {
+                    let qualify = |expr: &mut SqlExpr| {
+                        if let SqlExpr::Identifier(ident) = expr {
+                            let mut parts = source_qualifier.clone();
+                            parts.push(ident.clone());
+                            *expr = SqlExpr::CompoundIdentifier(parts);

Review Comment:
   This visitor qualifies every identifier, including identifiers that are not 
source columns. With `datafusion.sql_parser.dialect = 'duckdb'`, a NOT MATCHED 
insert containing `array_transform(s.arr, x -> x + 1)` is rewritten as 
`array_transform(s.arr, s.x -> s.x + 1)`. I verified with sqlparser 0.62.0 that 
reparsing the rewritten SQL produces a JSON-arrow expression instead of a 
lambda. Likewise, a session variable such as `@merge_value` becomes the column 
reference `s.@merge_value`. Please make the qualification scope-aware, 
preserving lambda parameters and variable references, and add regression tests 
for both cases.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to