JingsongLi commented on code in PR #1392:
URL: https://github.com/apache/paimon-rust/pull/1392#discussion_r4581496064


##########
crates/integrations/datafusion/src/delete.rs:
##########
@@ -63,14 +76,203 @@ pub(crate) async fn execute_delete(
         return execute_data_evolution_delete(ctx, delete, &table, 
table_ref).await;
     }
     if !schema.trimmed_primary_keys().is_empty() {
-        return Err(DataFusionError::Plan(
-            "DELETE on primary-key tables is not yet supported".to_string(),
-        ));
+        return execute_primary_key_delete(ctx, delete, &table, 
table_ref).await;
     }
 
     execute_cow_delete(ctx, delete, &table, table_ref).await
 }
 
+/// Execute DELETE on a deduplicate primary-key table by writing matched rows 
as DELETE records.
+async fn execute_primary_key_delete(
+    ctx: &SQLContext,
+    delete: &Delete,
+    table: &Table,
+    table_ref: &str,
+) -> DFResult<DataFrame> {
+    let options = table.schema().core_options();
+    if options.merge_engine().map_err(to_datafusion_error)? != 
MergeEngine::Deduplicate {
+        return Err(DataFusionError::NotImplemented(
+            "DELETE on primary-key tables requires 
merge-engine=deduplicate".into(),
+        ));
+    }
+    if options.ignore_delete() {
+        return Err(DataFusionError::NotImplemented(
+            "DELETE on primary-key tables requires ignore-delete=false".into(),
+        ));
+    }
+    // A masked key or sequence value cannot safely identify a tombstone 
target.
+    if options.query_auth_enabled() {
+        return Err(DataFusionError::NotImplemented(
+            "DELETE on query-authorized primary-key tables is not 
supported".into(),
+        ));
+    }
+    if options.rowkind_field().is_some() {
+        return Err(DataFusionError::NotImplemented(
+            "DELETE on primary-key tables with rowkind.field is not 
supported".into(),
+        ));
+    }
+    let from = match &delete.from {
+        FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables) 
=> tables,
+    };
+    if from.len() != 1
+        || !from[0].joins.is_empty()
+        || !delete.tables.is_empty()
+        || delete.using.is_some()
+        || delete.returning.is_some()
+        || delete.output.is_some()
+        || !delete.order_by.is_empty()
+        || delete.limit.is_some()
+        || !matches!(&from[0].relation,
+            TableFactor::Table {
+                args: None, version: None, with_ordinality: false,
+                json_path: None, sample: None, with_hints, partitions, 
index_hints, ..
+            } if with_hints.is_empty() && partitions.is_empty() && 
index_hints.is_empty())
+    {
+        return Err(DataFusionError::NotImplemented(
+            "Primary-key DELETE supports a single target with an optional 
WHERE clause".into(),
+        ));
+    }
+
+    let resources =
+        
crate::memory::writer_resources(&ctx.ctx().task_ctx()).map_err(to_datafusion_error)?;
+    let write_builder = table.new_write_builder().with_resources(resources);
+    let commit = write_builder
+        .try_new_commit()
+        .map_err(to_datafusion_error)?;
+    let Some(input) = plan_primary_key_delete_input(ctx, delete, table, 
table_ref).await? else {
+        return ok_result(ctx.ctx(), 0);
+    };
+    let mut writer = write_builder.new_write().map_err(to_datafusion_error)?;
+
+    let staged = async {
+        let mut stream = input.execute_stream().await?;
+        let mut matched_rows = 0;
+        while let Some(batch) = stream.next().await {
+            let batch = to_paimon_batch(batch?, false)?;
+            matched_rows += batch.num_rows() as u64;
+            writer
+                .write_arrow_batch(&with_delete_row_kind(batch)?)
+                .await
+                .map_err(to_datafusion_error)?;
+        }
+        let messages = 
writer.prepare_commit().await.map_err(to_datafusion_error)?;
+        Ok::<_, DataFusionError>((matched_rows, messages))
+    }
+    .await;
+    // Close unprepared output even on scan failure; prepared files are 
caller-owned.
+    writer.close().await;
+    let (matched_rows, messages) = staged?;
+    if !messages.is_empty() {
+        // Publication may have succeeded even on error. Keep prepared files 
for recovery.
+        commit.commit(messages).await.map_err(|error| {
+            let context = format!(
+                "Primary-key DELETE commit failed for table '{}' 
(commit_user={}, matched_rows={matched_rows}); prepared files retained",
+                table.identifier(),
+                write_builder.commit_user(),
+            );
+            log::warn!("{context}: {error}");
+            to_datafusion_error(error).context(context)
+        })?;
+    }
+    ok_result(ctx.ctx(), matched_rows)
+}
+
+/// Mark every row of `batch` as a DELETE record via the `_VALUE_KIND` column.
+fn with_delete_row_kind(batch: RecordBatch) -> DFResult<RecordBatch> {
+    let schema = batch.schema();
+    let mut fields = schema.fields().to_vec();
+    fields.push(Arc::new(Field::new(
+        VALUE_KIND_FIELD_NAME,
+        DataType::Int8,
+        false,
+    )));
+    let mut columns = batch.columns().to_vec();
+    columns.push(Arc::new(Int8Array::from(vec![
+        RowKind::Delete.to_value();
+        batch.num_rows()
+    ])));
+    Ok(RecordBatch::try_new(
+        Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone())),
+        columns,
+    )?)
+}
+
+/// Plan the target scan for the DELETE predicate, pinned to the latest 
snapshot.
+///
+/// Returns `None` when the table has no snapshot yet.
+async fn plan_primary_key_delete_input(
+    ctx: &SQLContext,
+    delete: &Delete,
+    table: &Table,
+    table_ref: &str,
+) -> DFResult<Option<DataFrame>> {
+    let snapshot = table
+        .resolve_read_snapshot()
+        .await
+        .map_err(to_datafusion_error)?;
+    let where_clause = delete
+        .selection
+        .as_ref()
+        .map(|predicate| format!(" WHERE {predicate}"))
+        .unwrap_or_default();
+    let matched = ctx
+        .ctx()
+        .sql(&format!("SELECT * FROM {table_ref}{where_clause}"))
+        .await?;
+    let is_target = |provider: &PaimonTableProvider| {
+        let source = provider.table();
+        source.identifier() == table.identifier()
+            && source.location() == table.location()
+            && source.branch() == table.branch()
+            && !source.is_time_traveled()
+    };
+    // Validate the direct FROM source: a view can synthesize keys or hide 
scans.
+    let source = match matched.logical_plan() {
+        LogicalPlan::Projection(projection) => projection.input.as_ref(),
+        plan => plan,
+    };
+    let source = match source {
+        LogicalPlan::Filter(filter) => filter.input.as_ref(),
+        plan => plan,
+    };
+    let target_changed = || DataFusionError::Plan("DELETE target changed 
during planning".into());
+    let LogicalPlan::TableScan(scan) = source else {
+        return Err(target_changed());
+    };
+    let provider = source_as_provider(&scan.source)?;
+    if !provider
+        .downcast_ref::<PaimonTableProvider>()
+        .is_some_and(is_target)
+    {
+        return Err(target_changed());
+    }
+    let Some(snapshot) = snapshot else {
+        return Ok(None);
+    };
+    // Self-references in predicates must use the captured target snapshot too.
+    let plan = matched
+        .into_unoptimized_plan()
+        .transform_up_with_subqueries(|plan| {
+            let LogicalPlan::TableScan(mut scan) = plan else {
+                return Ok(Transformed::no(plan));
+            };
+            let Ok(provider) = source_as_provider(&scan.source) else {
+                return Ok(Transformed::no(LogicalPlan::TableScan(scan)));
+            };
+            if let Some(provider) = 
provider.downcast_ref::<PaimonTableProvider>() {
+                if is_target(provider) {
+                    scan.source = provider_as_source(Arc::new(
+                        provider.clone().with_pinned_snapshot(&snapshot),

Review Comment:
   [P1] Merge uncompacted DV versions before evaluating DELETE
   
   Pinning the snapshot does not make this provider read all versions. On a PK 
DV table, `deletion-vectors.merge-on-read` defaults to false, so its ordinary 
query path skips level-0 files. The new DELETE then evaluates its predicate on 
stale compacted values and writes a newer tombstone for the wrong logical 
version.
   
   Public API reproduction (no concurrent writes): create a deduplicate PK 
table with bucket=2, write-only=true, deletion-vectors.enabled=true and default 
MOR=false; enable local-cache on the catalog for compaction. Insert 
`(id=1,v=10)` and compact it; then INSERT `(1,20)` without compacting. The 
captured snapshot already contains current value 20, but `DELETE FROM t WHERE 
v=10` reports one affected row. After the following compaction, key 1 is gone. 
A full MOR read before DELETE returns `(1,20)`, and the identical MOR=true 
control reports zero and preserves `(1,20)` after compaction. This is not the 
documented concurrent-upsert limitation: value 20 committed before DELETE began.
   
   Use the full all-files/key-merge DML scan path (including target 
self-references) before applying the predicate, as update inputs do, or reject 
DV tables unless the required MOR semantics are enabled. Merely pinning the 
normal compacted-only provider cannot safely identify tombstones. Add the 
compacted-old plus uncompacted-new predicate regression.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to