SteNicholas commented on code in PR #1392:
URL: https://github.com/apache/paimon-rust/pull/1392#discussion_r4582525177
##########
crates/integrations/datafusion/src/delete.rs:
##########
@@ -63,14 +76,203 @@ pub(crate) async fn execute_delete(
return execute_data_evolution_delete(ctx, delete, &table,
table_ref).await;
}
if !schema.trimmed_primary_keys().is_empty() {
- return Err(DataFusionError::Plan(
- "DELETE on primary-key tables is not yet supported".to_string(),
- ));
+ return execute_primary_key_delete(ctx, delete, &table,
table_ref).await;
}
execute_cow_delete(ctx, delete, &table, table_ref).await
}
+/// Execute DELETE on a deduplicate primary-key table by writing matched rows
as DELETE records.
+async fn execute_primary_key_delete(
+ ctx: &SQLContext,
+ delete: &Delete,
+ table: &Table,
+ table_ref: &str,
+) -> DFResult<DataFrame> {
+ let options = table.schema().core_options();
+ if options.merge_engine().map_err(to_datafusion_error)? !=
MergeEngine::Deduplicate {
+ return Err(DataFusionError::NotImplemented(
+ "DELETE on primary-key tables requires
merge-engine=deduplicate".into(),
+ ));
+ }
+ if options.ignore_delete() {
+ return Err(DataFusionError::NotImplemented(
+ "DELETE on primary-key tables requires ignore-delete=false".into(),
+ ));
+ }
+ // A masked key or sequence value cannot safely identify a tombstone
target.
+ if options.query_auth_enabled() {
+ return Err(DataFusionError::NotImplemented(
+ "DELETE on query-authorized primary-key tables is not
supported".into(),
+ ));
+ }
+ if options.rowkind_field().is_some() {
+ return Err(DataFusionError::NotImplemented(
+ "DELETE on primary-key tables with rowkind.field is not
supported".into(),
+ ));
+ }
+ let from = match &delete.from {
+ FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables)
=> tables,
+ };
+ if from.len() != 1
+ || !from[0].joins.is_empty()
+ || !delete.tables.is_empty()
+ || delete.using.is_some()
+ || delete.returning.is_some()
+ || delete.output.is_some()
+ || !delete.order_by.is_empty()
+ || delete.limit.is_some()
+ || !matches!(&from[0].relation,
+ TableFactor::Table {
+ args: None, version: None, with_ordinality: false,
+ json_path: None, sample: None, with_hints, partitions,
index_hints, ..
+ } if with_hints.is_empty() && partitions.is_empty() &&
index_hints.is_empty())
+ {
+ return Err(DataFusionError::NotImplemented(
+ "Primary-key DELETE supports a single target with an optional
WHERE clause".into(),
+ ));
+ }
+
+ let resources =
+
crate::memory::writer_resources(&ctx.ctx().task_ctx()).map_err(to_datafusion_error)?;
+ let write_builder = table.new_write_builder().with_resources(resources);
+ let commit = write_builder
+ .try_new_commit()
+ .map_err(to_datafusion_error)?;
+ let Some(input) = plan_primary_key_delete_input(ctx, delete, table,
table_ref).await? else {
+ return ok_result(ctx.ctx(), 0);
+ };
+ let mut writer = write_builder.new_write().map_err(to_datafusion_error)?;
+
+ let staged = async {
+ let mut stream = input.execute_stream().await?;
+ let mut matched_rows = 0;
+ while let Some(batch) = stream.next().await {
+ let batch = to_paimon_batch(batch?, false)?;
+ matched_rows += batch.num_rows() as u64;
+ writer
+ .write_arrow_batch(&with_delete_row_kind(batch)?)
+ .await
+ .map_err(to_datafusion_error)?;
+ }
+ let messages =
writer.prepare_commit().await.map_err(to_datafusion_error)?;
+ Ok::<_, DataFusionError>((matched_rows, messages))
+ }
+ .await;
+ // Close unprepared output even on scan failure; prepared files are
caller-owned.
+ writer.close().await;
+ let (matched_rows, messages) = staged?;
+ if !messages.is_empty() {
+ // Publication may have succeeded even on error. Keep prepared files
for recovery.
+ commit.commit(messages).await.map_err(|error| {
+ let context = format!(
+ "Primary-key DELETE commit failed for table '{}'
(commit_user={}, matched_rows={matched_rows}); prepared files retained",
+ table.identifier(),
+ write_builder.commit_user(),
+ );
+ log::warn!("{context}: {error}");
+ to_datafusion_error(error).context(context)
+ })?;
+ }
+ ok_result(ctx.ctx(), matched_rows)
+}
+
+/// Mark every row of `batch` as a DELETE record via the `_VALUE_KIND` column.
+fn with_delete_row_kind(batch: RecordBatch) -> DFResult<RecordBatch> {
+ let schema = batch.schema();
+ let mut fields = schema.fields().to_vec();
+ fields.push(Arc::new(Field::new(
+ VALUE_KIND_FIELD_NAME,
+ DataType::Int8,
+ false,
+ )));
+ let mut columns = batch.columns().to_vec();
+ columns.push(Arc::new(Int8Array::from(vec![
+ RowKind::Delete.to_value();
+ batch.num_rows()
+ ])));
+ Ok(RecordBatch::try_new(
+ Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone())),
+ columns,
+ )?)
+}
+
+/// Plan the target scan for the DELETE predicate, pinned to the latest
snapshot.
+///
+/// Returns `None` when the table has no snapshot yet.
+async fn plan_primary_key_delete_input(
+ ctx: &SQLContext,
+ delete: &Delete,
+ table: &Table,
+ table_ref: &str,
+) -> DFResult<Option<DataFrame>> {
+ let snapshot = table
+ .resolve_read_snapshot()
+ .await
+ .map_err(to_datafusion_error)?;
+ let where_clause = delete
+ .selection
+ .as_ref()
+ .map(|predicate| format!(" WHERE {predicate}"))
+ .unwrap_or_default();
+ let matched = ctx
+ .ctx()
+ .sql(&format!("SELECT * FROM {table_ref}{where_clause}"))
+ .await?;
+ let is_target = |provider: &PaimonTableProvider| {
+ let source = provider.table();
+ source.identifier() == table.identifier()
+ && source.location() == table.location()
+ && source.branch() == table.branch()
+ && !source.is_time_traveled()
+ };
+ // Validate the direct FROM source: a view can synthesize keys or hide
scans.
+ let source = match matched.logical_plan() {
+ LogicalPlan::Projection(projection) => projection.input.as_ref(),
+ plan => plan,
+ };
+ let source = match source {
+ LogicalPlan::Filter(filter) => filter.input.as_ref(),
+ plan => plan,
+ };
+ let target_changed = || DataFusionError::Plan("DELETE target changed
during planning".into());
+ let LogicalPlan::TableScan(scan) = source else {
+ return Err(target_changed());
+ };
+ let provider = source_as_provider(&scan.source)?;
+ if !provider
+ .downcast_ref::<PaimonTableProvider>()
+ .is_some_and(is_target)
+ {
+ return Err(target_changed());
+ }
+ let Some(snapshot) = snapshot else {
+ return Ok(None);
+ };
+ // Self-references in predicates must use the captured target snapshot too.
+ let plan = matched
+ .into_unoptimized_plan()
+ .transform_up_with_subqueries(|plan| {
+ let LogicalPlan::TableScan(mut scan) = plan else {
+ return Ok(Transformed::no(plan));
+ };
+ let Ok(provider) = source_as_provider(&scan.source) else {
+ return Ok(Transformed::no(LogicalPlan::TableScan(scan)));
+ };
+ if let Some(provider) =
provider.downcast_ref::<PaimonTableProvider>() {
+ if is_target(provider) {
+ scan.source = provider_as_source(Arc::new(
+ provider.clone().with_pinned_snapshot(&snapshot),
Review Comment:
Fixed in 2670b97b46bfeaf25d4d2aa874e1f5957950be26. The DELETE target and all
live self-references now use `with_primary_key_delete_snapshot`, which enables
`deletion-vectors.merge-on-read` on a snapshot-pinned copy before predicate
evaluation. This includes the uncompacted versions while preserving ordinary
query behavior and explicitly historical/incremental views.
`primary_key_delete_merges_uncompacted_dv_versions` covers the compacted
`(1, 10)` plus uncompacted `(1, 20)` reproduction with merge-on-read unset,
false and true: `WHERE v = 10` returns zero, publishes no snapshot, and
preserves `(1, 20)` after compaction. It also checks deletes matching updated
values, uncompacted tombstones and repeated deletes.
`primary_key_delete_subqueries_merge_uncompacted_dv_versions` covers
IN/scalar/correlated subqueries and live views, with historical-view
preservation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]