JingsongLi commented on code in PR #1392:
URL: https://github.com/apache/paimon-rust/pull/1392#discussion_r4581496064
##########
crates/integrations/datafusion/src/delete.rs:
##########
@@ -63,14 +76,203 @@ pub(crate) async fn execute_delete(
return execute_data_evolution_delete(ctx, delete, &table,
table_ref).await;
}
if !schema.trimmed_primary_keys().is_empty() {
- return Err(DataFusionError::Plan(
- "DELETE on primary-key tables is not yet supported".to_string(),
- ));
+ return execute_primary_key_delete(ctx, delete, &table,
table_ref).await;
}
execute_cow_delete(ctx, delete, &table, table_ref).await
}
+/// Execute DELETE on a deduplicate primary-key table by writing matched rows
as DELETE records.
+async fn execute_primary_key_delete(
+ ctx: &SQLContext,
+ delete: &Delete,
+ table: &Table,
+ table_ref: &str,
+) -> DFResult<DataFrame> {
+ let options = table.schema().core_options();
+ if options.merge_engine().map_err(to_datafusion_error)? !=
MergeEngine::Deduplicate {
+ return Err(DataFusionError::NotImplemented(
+ "DELETE on primary-key tables requires
merge-engine=deduplicate".into(),
+ ));
+ }
+ if options.ignore_delete() {
+ return Err(DataFusionError::NotImplemented(
+ "DELETE on primary-key tables requires ignore-delete=false".into(),
+ ));
+ }
+ // A masked key or sequence value cannot safely identify a tombstone
target.
+ if options.query_auth_enabled() {
+ return Err(DataFusionError::NotImplemented(
+ "DELETE on query-authorized primary-key tables is not
supported".into(),
+ ));
+ }
+ if options.rowkind_field().is_some() {
+ return Err(DataFusionError::NotImplemented(
+ "DELETE on primary-key tables with rowkind.field is not
supported".into(),
+ ));
+ }
+ let from = match &delete.from {
+ FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables)
=> tables,
+ };
+ if from.len() != 1
+ || !from[0].joins.is_empty()
+ || !delete.tables.is_empty()
+ || delete.using.is_some()
+ || delete.returning.is_some()
+ || delete.output.is_some()
+ || !delete.order_by.is_empty()
+ || delete.limit.is_some()
+ || !matches!(&from[0].relation,
+ TableFactor::Table {
+ args: None, version: None, with_ordinality: false,
+ json_path: None, sample: None, with_hints, partitions,
index_hints, ..
+ } if with_hints.is_empty() && partitions.is_empty() &&
index_hints.is_empty())
+ {
+ return Err(DataFusionError::NotImplemented(
+ "Primary-key DELETE supports a single target with an optional
WHERE clause".into(),
+ ));
+ }
+
+ let resources =
+
crate::memory::writer_resources(&ctx.ctx().task_ctx()).map_err(to_datafusion_error)?;
+ let write_builder = table.new_write_builder().with_resources(resources);
+ let commit = write_builder
+ .try_new_commit()
+ .map_err(to_datafusion_error)?;
+ let Some(input) = plan_primary_key_delete_input(ctx, delete, table,
table_ref).await? else {
+ return ok_result(ctx.ctx(), 0);
+ };
+ let mut writer = write_builder.new_write().map_err(to_datafusion_error)?;
+
+ let staged = async {
+ let mut stream = input.execute_stream().await?;
+ let mut matched_rows = 0;
+ while let Some(batch) = stream.next().await {
+ let batch = to_paimon_batch(batch?, false)?;
+ matched_rows += batch.num_rows() as u64;
+ writer
+ .write_arrow_batch(&with_delete_row_kind(batch)?)
+ .await
+ .map_err(to_datafusion_error)?;
+ }
+ let messages =
writer.prepare_commit().await.map_err(to_datafusion_error)?;
+ Ok::<_, DataFusionError>((matched_rows, messages))
+ }
+ .await;
+ // Close unprepared output even on scan failure; prepared files are
caller-owned.
+ writer.close().await;
+ let (matched_rows, messages) = staged?;
+ if !messages.is_empty() {
+ // Publication may have succeeded even on error. Keep prepared files
for recovery.
+ commit.commit(messages).await.map_err(|error| {
+ let context = format!(
+ "Primary-key DELETE commit failed for table '{}'
(commit_user={}, matched_rows={matched_rows}); prepared files retained",
+ table.identifier(),
+ write_builder.commit_user(),
+ );
+ log::warn!("{context}: {error}");
+ to_datafusion_error(error).context(context)
+ })?;
+ }
+ ok_result(ctx.ctx(), matched_rows)
+}
+
+/// Mark every row of `batch` as a DELETE record via the `_VALUE_KIND` column.
+fn with_delete_row_kind(batch: RecordBatch) -> DFResult<RecordBatch> {
+ let schema = batch.schema();
+ let mut fields = schema.fields().to_vec();
+ fields.push(Arc::new(Field::new(
+ VALUE_KIND_FIELD_NAME,
+ DataType::Int8,
+ false,
+ )));
+ let mut columns = batch.columns().to_vec();
+ columns.push(Arc::new(Int8Array::from(vec![
+ RowKind::Delete.to_value();
+ batch.num_rows()
+ ])));
+ Ok(RecordBatch::try_new(
+ Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone())),
+ columns,
+ )?)
+}
+
+/// Plan the target scan for the DELETE predicate, pinned to the latest
snapshot.
+///
+/// Returns `None` when the table has no snapshot yet.
+async fn plan_primary_key_delete_input(
+ ctx: &SQLContext,
+ delete: &Delete,
+ table: &Table,
+ table_ref: &str,
+) -> DFResult<Option<DataFrame>> {
+ let snapshot = table
+ .resolve_read_snapshot()
+ .await
+ .map_err(to_datafusion_error)?;
+ let where_clause = delete
+ .selection
+ .as_ref()
+ .map(|predicate| format!(" WHERE {predicate}"))
+ .unwrap_or_default();
+ let matched = ctx
+ .ctx()
+ .sql(&format!("SELECT * FROM {table_ref}{where_clause}"))
+ .await?;
+ let is_target = |provider: &PaimonTableProvider| {
+ let source = provider.table();
+ source.identifier() == table.identifier()
+ && source.location() == table.location()
+ && source.branch() == table.branch()
+ && !source.is_time_traveled()
+ };
+ // Validate the direct FROM source: a view can synthesize keys or hide
scans.
+ let source = match matched.logical_plan() {
+ LogicalPlan::Projection(projection) => projection.input.as_ref(),
+ plan => plan,
+ };
+ let source = match source {
+ LogicalPlan::Filter(filter) => filter.input.as_ref(),
+ plan => plan,
+ };
+ let target_changed = || DataFusionError::Plan("DELETE target changed
during planning".into());
+ let LogicalPlan::TableScan(scan) = source else {
+ return Err(target_changed());
+ };
+ let provider = source_as_provider(&scan.source)?;
+ if !provider
+ .downcast_ref::<PaimonTableProvider>()
+ .is_some_and(is_target)
+ {
+ return Err(target_changed());
+ }
+ let Some(snapshot) = snapshot else {
+ return Ok(None);
+ };
+ // Self-references in predicates must use the captured target snapshot too.
+ let plan = matched
+ .into_unoptimized_plan()
+ .transform_up_with_subqueries(|plan| {
+ let LogicalPlan::TableScan(mut scan) = plan else {
+ return Ok(Transformed::no(plan));
+ };
+ let Ok(provider) = source_as_provider(&scan.source) else {
+ return Ok(Transformed::no(LogicalPlan::TableScan(scan)));
+ };
+ if let Some(provider) =
provider.downcast_ref::<PaimonTableProvider>() {
+ if is_target(provider) {
+ scan.source = provider_as_source(Arc::new(
+ provider.clone().with_pinned_snapshot(&snapshot),
Review Comment:
[P1] Merge uncompacted DV versions before evaluating DELETE
Pinning the snapshot does not make this provider read all versions. On a PK
DV table, `deletion-vectors.merge-on-read` defaults to false, so its ordinary
query path skips level-0 files. The new DELETE then evaluates its predicate on
stale compacted values and writes a newer tombstone for the wrong logical
version.
Public API reproduction (no concurrent writes): create a deduplicate PK
table with bucket=2, write-only=true, deletion-vectors.enabled=true and default
MOR=false; enable local-cache on the catalog for compaction. Insert
`(id=1,v=10)` and compact it; then INSERT `(1,20)` without compacting. The
captured snapshot already contains current value 20, but `DELETE FROM t WHERE
v=10` reports one affected row. After the following compaction, key 1 is gone.
A full MOR read before DELETE returns `(1,20)`, and the identical MOR=true
control reports zero and preserves `(1,20)` after compaction. This is not the
documented concurrent-upsert limitation: value 20 committed before DELETE began.
Use the full all-files/key-merge DML scan path (including target
self-references) before applying the predicate, as update inputs do, or reject
DV tables unless the required MOR semantics are enabled. Merely pinning the
normal compacted-only provider cannot safely identify tombstones. Add the
compacted-old plus uncompacted-new predicate regression.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]