SteNicholas commented on code in PR #1392:
URL: https://github.com/apache/paimon-rust/pull/1392#discussion_r4582525177


##########
crates/integrations/datafusion/src/delete.rs:
##########
@@ -63,14 +76,203 @@ pub(crate) async fn execute_delete(
         return execute_data_evolution_delete(ctx, delete, &table, 
table_ref).await;
     }
     if !schema.trimmed_primary_keys().is_empty() {
-        return Err(DataFusionError::Plan(
-            "DELETE on primary-key tables is not yet supported".to_string(),
-        ));
+        return execute_primary_key_delete(ctx, delete, &table, 
table_ref).await;
     }
 
     execute_cow_delete(ctx, delete, &table, table_ref).await
 }
 
+/// Execute DELETE on a deduplicate primary-key table by writing matched rows 
as DELETE records.
+async fn execute_primary_key_delete(
+    ctx: &SQLContext,
+    delete: &Delete,
+    table: &Table,
+    table_ref: &str,
+) -> DFResult<DataFrame> {
+    let options = table.schema().core_options();
+    if options.merge_engine().map_err(to_datafusion_error)? != 
MergeEngine::Deduplicate {
+        return Err(DataFusionError::NotImplemented(
+            "DELETE on primary-key tables requires 
merge-engine=deduplicate".into(),
+        ));
+    }
+    if options.ignore_delete() {
+        return Err(DataFusionError::NotImplemented(
+            "DELETE on primary-key tables requires ignore-delete=false".into(),
+        ));
+    }
+    // A masked key or sequence value cannot safely identify a tombstone 
target.
+    if options.query_auth_enabled() {
+        return Err(DataFusionError::NotImplemented(
+            "DELETE on query-authorized primary-key tables is not 
supported".into(),
+        ));
+    }
+    if options.rowkind_field().is_some() {
+        return Err(DataFusionError::NotImplemented(
+            "DELETE on primary-key tables with rowkind.field is not 
supported".into(),
+        ));
+    }
+    let from = match &delete.from {
+        FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables) 
=> tables,
+    };
+    if from.len() != 1
+        || !from[0].joins.is_empty()
+        || !delete.tables.is_empty()
+        || delete.using.is_some()
+        || delete.returning.is_some()
+        || delete.output.is_some()
+        || !delete.order_by.is_empty()
+        || delete.limit.is_some()
+        || !matches!(&from[0].relation,
+            TableFactor::Table {
+                args: None, version: None, with_ordinality: false,
+                json_path: None, sample: None, with_hints, partitions, 
index_hints, ..
+            } if with_hints.is_empty() && partitions.is_empty() && 
index_hints.is_empty())
+    {
+        return Err(DataFusionError::NotImplemented(
+            "Primary-key DELETE supports a single target with an optional 
WHERE clause".into(),
+        ));
+    }
+
+    let resources =
+        
crate::memory::writer_resources(&ctx.ctx().task_ctx()).map_err(to_datafusion_error)?;
+    let write_builder = table.new_write_builder().with_resources(resources);
+    let commit = write_builder
+        .try_new_commit()
+        .map_err(to_datafusion_error)?;
+    let Some(input) = plan_primary_key_delete_input(ctx, delete, table, 
table_ref).await? else {
+        return ok_result(ctx.ctx(), 0);
+    };
+    let mut writer = write_builder.new_write().map_err(to_datafusion_error)?;
+
+    let staged = async {
+        let mut stream = input.execute_stream().await?;
+        let mut matched_rows = 0;
+        while let Some(batch) = stream.next().await {
+            let batch = to_paimon_batch(batch?, false)?;
+            matched_rows += batch.num_rows() as u64;
+            writer
+                .write_arrow_batch(&with_delete_row_kind(batch)?)
+                .await
+                .map_err(to_datafusion_error)?;
+        }
+        let messages = 
writer.prepare_commit().await.map_err(to_datafusion_error)?;
+        Ok::<_, DataFusionError>((matched_rows, messages))
+    }
+    .await;
+    // Close unprepared output even on scan failure; prepared files are 
caller-owned.
+    writer.close().await;
+    let (matched_rows, messages) = staged?;
+    if !messages.is_empty() {
+        // Publication may have succeeded even on error. Keep prepared files 
for recovery.
+        commit.commit(messages).await.map_err(|error| {
+            let context = format!(
+                "Primary-key DELETE commit failed for table '{}' 
(commit_user={}, matched_rows={matched_rows}); prepared files retained",
+                table.identifier(),
+                write_builder.commit_user(),
+            );
+            log::warn!("{context}: {error}");
+            to_datafusion_error(error).context(context)
+        })?;
+    }
+    ok_result(ctx.ctx(), matched_rows)
+}
+
+/// Mark every row of `batch` as a DELETE record via the `_VALUE_KIND` column.
+fn with_delete_row_kind(batch: RecordBatch) -> DFResult<RecordBatch> {
+    let schema = batch.schema();
+    let mut fields = schema.fields().to_vec();
+    fields.push(Arc::new(Field::new(
+        VALUE_KIND_FIELD_NAME,
+        DataType::Int8,
+        false,
+    )));
+    let mut columns = batch.columns().to_vec();
+    columns.push(Arc::new(Int8Array::from(vec![
+        RowKind::Delete.to_value();
+        batch.num_rows()
+    ])));
+    Ok(RecordBatch::try_new(
+        Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone())),
+        columns,
+    )?)
+}
+
+/// Plan the target scan for the DELETE predicate, pinned to the latest 
snapshot.
+///
+/// Returns `None` when the table has no snapshot yet.
+async fn plan_primary_key_delete_input(
+    ctx: &SQLContext,
+    delete: &Delete,
+    table: &Table,
+    table_ref: &str,
+) -> DFResult<Option<DataFrame>> {
+    let snapshot = table
+        .resolve_read_snapshot()
+        .await
+        .map_err(to_datafusion_error)?;
+    let where_clause = delete
+        .selection
+        .as_ref()
+        .map(|predicate| format!(" WHERE {predicate}"))
+        .unwrap_or_default();
+    let matched = ctx
+        .ctx()
+        .sql(&format!("SELECT * FROM {table_ref}{where_clause}"))
+        .await?;
+    let is_target = |provider: &PaimonTableProvider| {
+        let source = provider.table();
+        source.identifier() == table.identifier()
+            && source.location() == table.location()
+            && source.branch() == table.branch()
+            && !source.is_time_traveled()
+    };
+    // Validate the direct FROM source: a view can synthesize keys or hide 
scans.
+    let source = match matched.logical_plan() {
+        LogicalPlan::Projection(projection) => projection.input.as_ref(),
+        plan => plan,
+    };
+    let source = match source {
+        LogicalPlan::Filter(filter) => filter.input.as_ref(),
+        plan => plan,
+    };
+    let target_changed = || DataFusionError::Plan("DELETE target changed 
during planning".into());
+    let LogicalPlan::TableScan(scan) = source else {
+        return Err(target_changed());
+    };
+    let provider = source_as_provider(&scan.source)?;
+    if !provider
+        .downcast_ref::<PaimonTableProvider>()
+        .is_some_and(is_target)
+    {
+        return Err(target_changed());
+    }
+    let Some(snapshot) = snapshot else {
+        return Ok(None);
+    };
+    // Self-references in predicates must use the captured target snapshot too.
+    let plan = matched
+        .into_unoptimized_plan()
+        .transform_up_with_subqueries(|plan| {
+            let LogicalPlan::TableScan(mut scan) = plan else {
+                return Ok(Transformed::no(plan));
+            };
+            let Ok(provider) = source_as_provider(&scan.source) else {
+                return Ok(Transformed::no(LogicalPlan::TableScan(scan)));
+            };
+            if let Some(provider) = 
provider.downcast_ref::<PaimonTableProvider>() {
+                if is_target(provider) {
+                    scan.source = provider_as_source(Arc::new(
+                        provider.clone().with_pinned_snapshot(&snapshot),

Review Comment:
   Fixed in 2670b97b46bfeaf25d4d2aa874e1f5957950be26. The DELETE target and all 
live self-references now use `with_primary_key_delete_snapshot`, which enables 
`deletion-vectors.merge-on-read` on a snapshot-pinned copy before predicate 
evaluation. This includes the uncompacted versions while preserving ordinary 
query behavior and explicitly historical/incremental views.
   
   `primary_key_delete_merges_uncompacted_dv_versions` covers the compacted 
`(1, 10)` plus uncompacted `(1, 20)` reproduction with merge-on-read unset, 
false and true: `WHERE v = 10` returns zero, publishes no snapshot, and 
preserves `(1, 20)` after compaction. It also checks deletes matching updated 
values, uncompacted tombstones and repeated deletes. 
`primary_key_delete_subqueries_merge_uncompacted_dv_versions` covers 
IN/scalar/correlated subqueries and live views, with historical-view 
preservation.
   
   All 486 locally run tests passed on Rust 1.98.0 with the 16 MiB stack, 
including the 18 PK DELETE integration tests. Formatting and Clippy passed as 
well; the commands and coverage are recorded in the updated PR description.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to