JingsongLi commented on code in PR #11174:
URL: https://github.com/apache/paimon/pull/11174#discussion_r4624295894


##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/ReplaceByKeysSource.scala:
##########
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.commands
+
+import org.apache.paimon.{CoreOptions, Snapshot}
+import org.apache.paimon.fs.Path
+import org.apache.paimon.manifest.PartitionEntry
+import org.apache.paimon.predicate.Predicate
+import org.apache.paimon.schema.TableSchema
+import 
org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine
+import org.apache.paimon.spark.{PaimonScanBuilder, SparkTable}
+import org.apache.paimon.table.{ChainGroupReadTable, DelegatedFileStoreTable, 
FallbackReadFileStoreTable, FileStoreTable}
+import org.apache.paimon.table.source.{DataFilePlan, DataTableScan, Split, 
TableScan}
+import org.apache.paimon.table.source.snapshot.TimeTravelUtil
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.optimizer.ComputeCurrentTime
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Command, 
CTERelationRef, Deduplicate, GlobalLimit, LocalLimit, LocalRelation, 
LogicalPlan, OneRowRelation, ParsedStatement, Range, Sample, Tail, Window}
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
+import org.apache.spark.sql.paimon.shims.SparkShimLoader
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+import java.util.{Collections, Map => JMap}
+
+import scala.collection.mutable
+
+/** A repeatable source for both replacement preparation jobs; no source rows 
or splits are cached. */
+private[spark] object ReplaceByKeysSource {
+  def bind(
+      spark: SparkSession,
+      table: FileStoreTable,
+      sourceQuery: String,
+      targetSnapshot: Snapshot): LogicalPlan = {
+    require(sourceQuery != null && sourceQuery.trim.nonEmpty, "source_query 
must not be empty")
+    val parsed = spark.sessionState.sqlParser.parsePlan(sourceQuery)
+    parsed.foreach {
+      case _: Command | _: ParsedStatement =>
+        throw new IllegalArgumentException("source_query must be a query, not 
a command")
+      case _ =>
+    }
+    // Dataset construction eagerly executes commands. Analyze without 
creating one first.
+    val analyzed = spark.sessionState.executePlan(parsed).analyzed
+    analyzed.foreach {
+      case _: Command =>
+        throw new IllegalArgumentException("source_query must be a query, not 
a command")
+      case _ =>
+    }
+    require(!analyzed.isStreaming, "Streaming source queries are not 
supported")
+    val selectors = Seq(
+      CoreOptions.SCAN_SNAPSHOT_ID.key(),
+      CoreOptions.SCAN_TAG_NAME.key(),
+      CoreOptions.SCAN_WATERMARK.key(),
+      CoreOptions.SCAN_TIMESTAMP.key(),
+      CoreOptions.SCAN_TIMESTAMP_MILLIS.key(),
+      CoreOptions.SCAN_VERSION.key()
+    )
+    val snapshots = mutable.Map
+      .empty[SnapshotKey, Option[Snapshot]]
+    if (!table.isInstanceOf[FallbackReadFileStoreTable])
+      snapshots.put(
+        (
+          table.location().toString,
+          table.coreOptions().branch(),
+          selectors.map(k => Option(table.options().get(k))),
+          CoreOptions.StartupMode.LATEST_FULL),
+        Option(targetSnapshot)
+      )
+    // Query-time values must not be evaluated afresh by the two preparation 
jobs.
+    ComputeCurrentTime(analyzed).transformUpWithSubqueries {
+      case r: DataSourceV2Relation =>
+        require(r.expressions.forall(_.deterministic), "Source query must be 
deterministic")
+        r.table match {
+          case SparkTable(fileTable: FileStoreTable) =>
+            val effective = 
fileTable.copyWithoutTimeTravel(r.options.asCaseSensitiveMap())
+            val mode = effective.coreOptions().startupMode()
+            require(
+              Set(
+                CoreOptions.StartupMode.LATEST_FULL,
+                CoreOptions.StartupMode.LATEST,
+                CoreOptions.StartupMode.FROM_TIMESTAMP,
+                CoreOptions.StartupMode.FROM_SNAPSHOT,
+                CoreOptions.StartupMode.FROM_SNAPSHOT_FULL
+              ).contains(mode),
+              "Source tables must use a full snapshot scan, not an incremental 
or file-time scan"
+            )
+            require(
+              
effective.coreOptions().toConfiguration.get(CoreOptions.BATCH_SCAN_MODE) ==
+                CoreOptions.BatchScanMode.NONE,
+              "Source monitoring scans are not supported"
+            )
+            val normalizedMode = mode match {
+              case CoreOptions.StartupMode.LATEST => 
CoreOptions.StartupMode.LATEST_FULL
+              case CoreOptions.StartupMode.FROM_SNAPSHOT_FULL =>
+                CoreOptions.StartupMode.FROM_SNAPSHOT
+              case other => other
+            }
+            val historicalKeys =
+              if (normalizedMode == CoreOptions.StartupMode.FROM_TIMESTAMP)
+                Seq(CoreOptions.SCAN_TIMESTAMP.key(), 
CoreOptions.SCAN_TIMESTAMP_MILLIS.key())
+              else
+                selectors.filterNot(
+                  k =>
+                    k == CoreOptions.SCAN_TIMESTAMP.key() ||
+                      k == CoreOptions.SCAN_TIMESTAMP_MILLIS.key())
+            require(
+              normalizedMode == CoreOptions.StartupMode.LATEST_FULL ||
+                historicalKeys.exists(k => effective.options().get(k) != null),
+              "Historical source mode requires an explicit matching snapshot 
selector"
+            )
+            val selected = bindTable(effective, selectors, snapshots)
+            val scanTable = new PinnedSourceTable(selected.table)
+            SparkShimLoader.shim.copyDataSourceV2Relation(r, scanTable, 
r.output)
+          case _ =>
+            throw new IllegalArgumentException("Source tables must support 
Paimon snapshots")
+        }
+      case p =>
+        require(p.expressions.forall(_.deterministic), "Source query must be 
deterministic")
+        val unstable = p match {
+          case _: Aggregate | _: Deduplicate | _: GlobalLimit | _: LocalLimit 
| _: Sample |
+              _: Tail | _: Window =>
+            true
+          // OFFSET is unavailable in older Spark compilation targets.
+          case _ => p.nodeName == "Offset"
+        }
+        require(
+          !unstable,
+          "Source query must be repeatable across projections; materialize 
aggregate, window, " +
+            "limit, offset, deduplication or sampled batches into a Paimon 
table first"
+        )
+        require(
+          p.children.nonEmpty || p.isInstanceOf[LocalRelation] || 
p.isInstanceOf[CTERelationRef] ||
+            p.isInstanceOf[OneRowRelation] || p.isInstanceOf[Range],
+          "Source leaves must be Paimon tables or immutable SQL values"
+        )
+        p
+    }
+  }
+
+  private type SnapshotKey = (String, String, Seq[Option[String]], 
CoreOptions.StartupMode)
+
+  private case class BoundRead(table: FileStoreTable)
+
+  // Share only Snapshot selections: each alias keeps its own analyzed schema 
and read options.
+  private def bindTable(
+      table: FileStoreTable,
+      selectors: Seq[String],
+      snapshots: mutable.Map[SnapshotKey, Option[Snapshot]]): BoundRead = {
+    require(
+      !table.isInstanceOf[ChainGroupReadTable],
+      "Chain-group replacement sources are not supported")
+    table match {
+      case fallback: FallbackReadFileStoreTable =>
+        // Keep Core's partition-level precedence and fallback reader. Pin 
every component now,
+        // including an empty current branch, rather than treating the 
composite as empty.
+        val components = Seq(fallback.wrapped(), fallback.other())
+          .filterNot {
+            component =>
+              // Core represents an absent historical fallback branch with 
this sentinel and skips it.
+              (component eq fallback.other()) &&
+              
Option(component.coreOptions().scanSnapshotId()).exists(_.longValue() == -1L)
+          }
+          .map(component => component -> bindTable(component, selectors, 
snapshots))
+        BoundRead(
+          new PinnedReadTable(
+            fallback,
+            () =>
+              fallback.newFallbackScan(
+                new java.util.function.Function[FileStoreTable, DataTableScan] 
{
+                  override def apply(component: FileStoreTable): DataTableScan 
=
+                    components.find(_._1 eq component).get._2.table.newScan()
+                })
+          ))
+      case _ =>
+        require(
+          table.coreOptions().toConfiguration.get(CoreOptions.BATCH_SCAN_MODE) 
==
+            CoreOptions.BatchScanMode.NONE,
+          "Source monitoring scans are not supported")
+        require(
+          Set(
+            CoreOptions.StartupMode.LATEST_FULL,
+            CoreOptions.StartupMode.LATEST,
+            CoreOptions.StartupMode.FROM_TIMESTAMP,
+            CoreOptions.StartupMode.FROM_SNAPSHOT,
+            CoreOptions.StartupMode.FROM_SNAPSHOT_FULL
+          ).contains(table.coreOptions().startupMode()),
+          "Source tables must use a full snapshot scan, not an incremental or 
file-time scan"
+        )
+        val mode = table.coreOptions().startupMode() match {
+          case CoreOptions.StartupMode.LATEST => 
CoreOptions.StartupMode.LATEST_FULL
+          case CoreOptions.StartupMode.FROM_SNAPSHOT_FULL => 
CoreOptions.StartupMode.FROM_SNAPSHOT
+          case other => other
+        }
+        val key = (
+          table.location().toString,
+          table.coreOptions().branch(),
+          selectors.map(k => Option(table.options().get(k))),
+          mode)
+        val snapshot = snapshots.getOrElseUpdate(
+          key,
+          if (mode == CoreOptions.StartupMode.LATEST_FULL)
+            Option(table.snapshotManager().latestSnapshot())
+          else {
+            val resolved = TimeTravelUtil.tryTravelToSnapshot(table)
+            require(resolved.isPresent, "Could not bind the requested source 
snapshot")
+            Some(resolved.get())
+          }
+        )
+        ordinary(table, snapshot)
+    }
+  }
+
+  private def ordinary(table: FileStoreTable, snapshot: Option[Snapshot]): 
BoundRead = {
+    val options = new java.util.HashMap[String, String]()
+    Seq(
+      CoreOptions.SCAN_SNAPSHOT_ID,
+      CoreOptions.SCAN_TAG_NAME,
+      CoreOptions.SCAN_WATERMARK,
+      CoreOptions.SCAN_TIMESTAMP,
+      CoreOptions.SCAN_TIMESTAMP_MILLIS,
+      CoreOptions.SCAN_VERSION
+    )
+      .foreach(k => options.put(k.key(), null))
+    options.put(CoreOptions.SCAN_MODE.key(), 
CoreOptions.StartupMode.FROM_SNAPSHOT.toString)

Review Comment:
   [P2] A fresh, empty source table makes the public CALL fail instead of 
producing an empty replacement. I reproduced this against both an ordinary 
append source and a postpone-bucket PK source with merge-on-read. A normal 
SELECT returns zero rows, but the CALL on a real target throws `must set only 
one key in [scan.snapshot-id,scan.tag-name,scan.watermark] when you use 
from-snapshot for scan.mode`. `ordinary` installs FROM_SNAPSHOT before the 
match; the None path has no selector and the delegated table still validates 
its options during Spark relation planning. Keep a valid configuration for the 
empty case and ensure its scan cannot see a commit made after the binding. Add 
a public CALL test with an unwritten source.



##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/ReplaceByKeysCommand.scala:
##########
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.commands
+
+import org.apache.paimon.{CoreOptions, Snapshot}
+import org.apache.paimon.spark.SparkTable
+import org.apache.paimon.spark.catalyst.analysis.PaimonOutputResolver
+import org.apache.paimon.table.FileStoreTable
+import org.apache.paimon.table.sink.{CommitMessage, CommitMessageImpl}
+import org.apache.paimon.table.source.DataSplit
+
+import org.apache.spark.sql.{DataFrame, Row, SparkSession}
+import org.apache.spark.sql.PaimonUtils.createDataset
+import org.apache.spark.sql.catalyst.expressions.{Attribute, 
SubqueryExpression}
+import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
+import org.apache.spark.sql.functions.{col, expr}
+
+import java.util.{Collections, TreeMap}
+
+import scala.collection.JavaConverters._
+
+/** Prepare a key-based delete and append, publishing both in one data commit. 
*/
+case class ReplaceByKeysCommand(
+    override val table: FileStoreTable,
+    relation: DataSourceV2Relation,
+    sourceQuery: String,
+    keyColumns: Seq[String],
+    targetFilter: Option[String])
+  extends PaimonRowLevelCommand {
+
+  override def run(spark: SparkSession): Seq[Row] = {
+    require(coreOptions.dataEvolutionEnabled(), "replace_by_keys requires a 
Data Evolution table")
+    require(
+      coreOptions.deletionVectorsEnabled(),
+      "replace_by_keys requires deletion-vectors.enabled=true")
+    val (messages, readSnapshotId) = prepare(spark)
+    val checked =
+      if (readSnapshotId < 0) messages
+      else 
messages.map(_.asInstanceOf[CommitMessageImpl].withCheckFromSnapshot(readSnapshotId))
+    // Once publication is attempted an exception can have an unknown outcome. 
Do not abort it.
+    writer.commit(checked, Snapshot.Operation.MERGE)
+    Seq.empty
+  }
+
+  private[spark] def prepare(spark: SparkSession): (Seq[CommitMessage], Long) 
= {
+    val snapshot = table.snapshotManager().latestSnapshot()
+    def byName(attributes: Seq[Attribute]): TreeMap[String, Attribute] = {
+      val result =
+        if (spark.sessionState.conf.caseSensitiveAnalysis) new TreeMap[String, 
Attribute]()
+        else new TreeMap[String, Attribute](String.CASE_INSENSITIVE_ORDER)
+      attributes.foreach {
+        attribute =>
+          require(!result.containsKey(attribute.name), s"Ambiguous column 
name: ${attribute.name}")
+          result.put(attribute.name, attribute)
+      }
+      result
+    }
+    val fields = byName(relation.output)
+    require(keyColumns.nonEmpty, "key_columns must not be empty")
+    val keys = keyColumns.map {
+      key =>
+        require(fields.containsKey(key), s"Unknown target key column: $key")
+        fields.get(key).name
+    }
+    require(keys.distinct.size == keyColumns.size, "Duplicate key columns")
+    val source = ReplaceByKeysSource.bind(spark, table, sourceQuery, snapshot)
+    val sourceFields = byName(source.output)
+    require(
+      sourceFields.keySet() == fields.keySet(),
+      "Source query must supply all target columns by name")
+    // Reuse INSERT's assignment policy, nested/nullability checks and 
CHAR/VARCHAR handling.
+    val aligned = PaimonOutputResolver.resolveOutputColumns(
+      table.name(),
+      relation.output,
+      source,
+      byName = true,
+      mergeSchemaEnabled = false)
+    val input = createDataset(spark, aligned)
+    // Match original query keys: an INSERT cast must not change which old 
rows are deleted.
+    val sourceKeys = createDataset(spark, source)
+      .select(keys.map(k => col(quoted(sourceFields.get(k).name)).as(k)): _*)
+      .alias("source_keys")
+
+    var staged = Seq.empty[CommitMessage]
+    try {
+      staged = prepareDeletion(spark, snapshot, sourceKeys, keys)
+      staged = staged ++ writer.write(input)
+      (staged, if (snapshot == null) -1L else snapshot.id())
+    } catch {
+      case failure: Throwable =>
+        if (staged.nonEmpty) {
+          try {
+            val cleanup = writer.writeBuilder.newCommit()
+            try cleanup.abort(staged.asJava)
+            finally cleanup.close()
+          } catch {
+            case cleanupFailure: Throwable =>
+              if (cleanupFailure ne failure) 
failure.addSuppressed(cleanupFailure)
+          }
+        }
+        throw failure
+    }
+  }
+
+  // Release the target scan and full file inventory before starting the 
append job.
+  private[spark] def prepareDeletion(
+      spark: SparkSession,
+      snapshot: Snapshot,
+      sourceKeys: DataFrame,
+      keys: Seq[String]): Seq[CommitMessage] = {
+    val predicate = targetFilter
+      .map {
+        text =>
+          val parsed = spark.sessionState.sqlParser.parseExpression(text)
+          require(
+            !SubqueryExpression.hasSubquery(parsed),
+            "target_filter must be a deterministic predicate over target 
columns")
+          val resolved = createDataset(spark, relation)
+            .filter(expr(text))
+            .queryExecution
+            .analyzed
+            .collect { case f: 
org.apache.spark.sql.catalyst.plans.logical.Filter => f.condition }
+            .head
+          require(resolved.deterministic, "target_filter must be 
deterministic")
+          resolved
+      }
+      .getOrElse(TrueLiteral)
+    // An empty baseline must not start reading a concurrent writer's first 
snapshot.
+    if (snapshot == null) return Seq.empty
+    val reader = table.newSnapshotReader().withSnapshot(snapshot)
+    convertConditionToPaimonPredicate(
+      predicate,
+      relation.output,
+      rowType,
+      ignorePartialFailure = true)
+      .foreach(reader.withFilter)
+    if (coreOptions.manifestDeleteFileDropStats()) reader.dropStats()
+    val splits = reader.read().splits().asScala.collect { case d: DataSplit => 
d }.toSeq
+    if (splits.nonEmpty) {
+      val rowId = (relation.output ++ relation.metadataOutput).find(_.name == 
"_ROW_ID").get
+      val scan = createNewScanPlan(splits, relation).copy(output = 
relation.output :+ rowId)
+      val target = createDataset(spark, 
scan).filter(toColumn(predicate)).alias("target")
+      val matches = keys
+        .map(key => col("target." + quoted(key)) === col("source_keys." + 
quoted(key)))
+        .reduce(_ && _)
+      // Semi join has the same TRUE-only membership semantics as DELETE WHERE 
key IN source.
+      val rowIds = target.join(sourceKeys, matches, 
"left_semi").select(col("target._ROW_ID"))

Review Comment:
   [P1] A semi join with `=` can delete rows that `DELETE ... IN (...)` 
retains. On actual Spark 3.5.9 Data Evolution/DV tables with 
`spark.sql.ansi.enabled=false` and `spark.sql.storeAssignmentPolicy=LEGACY`, 
seed `(1,'old')` in an INT-key target and use a source containing STRING 
`('01','new')`. The documented `DELETE WHERE id IN (SELECT id FROM source)` 
followed by `INSERT` leaves `(1,'old'), (1,'new')`. The public CALL reports 
success and leaves only `(1,'new')`. Catalyst coerces `=` and `InSubquery` 
differently under these configs; using the original source type alone does not 
preserve membership. Please use Spark's IN-subquery coercion, or reject 
incompatible key types before any destructive commit, and add this end-to-end 
regression.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to