zhuxiangyi commented on code in PR #10105: URL: https://github.com/apache/paimon/pull/10105#discussion_r4378711010
########## paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/StreamingWriteContext.scala: ########## @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.write + +import org.apache.paimon.io.{CompactIncrement, DataIncrement} +import org.apache.paimon.table.FileStoreTable +import org.apache.paimon.table.sink.{CommitMessage, CommitMessageImpl, InnerTableCommit} + +import org.apache.spark.internal.Logging + +import java.util.Collections + +import scala.collection.JavaConverters._ + +/** + * The committer of one run of a streaming query. It is created with the first micro-batch of the + * run and kept until the query terminates, the way a Flink committer lives across checkpoints: the + * maintenance a commit starts (tag creation, partition and snapshot expiration) can run on while + * the next micro-batch is written, and only a micro-batch that may be a replay has to look up what + * a previous run committed. + * + * @param commitUser + * the commit user of the query, stable across its runs, under which every micro-batch is + * committed so that a replayed one can be recognised. + * @param marker + * where the query records what it is about to commit, when it has a checkpoint location to keep + * it in. + */ +class StreamingWriteContext(val commitUser: String, @transient marker: Option[CommitMarker]) + extends AutoCloseable + with Serializable + with Logging { + + /** Driver-side state; the context itself travels with the write closures. */ + @transient private var committer: InnerTableCommit = _ + + /** The identifier of the last micro-batch this run committed, none before the first one. */ + @transient private var lastCommitted: Option[Long] = None + + /** The committer of the run, created by `create` on first use and kept afterwards. */ + def openCommitter(create: => InnerTableCommit): InnerTableCommit = + synchronized { + if (committer == null) { + committer = create + } + committer + } + + /** + * Commits a micro-batch under its identifier. + * + * Structured Streaming replays a micro-batch with its original batch id when a query is restarted + * after failing between the sink returning and Spark recording the batch as completed, so the + * first micro-batch of a run may already have been committed by a previous run. A batch that + * follows one this run committed cannot be a replay of it, and is committed without looking the + * history of the commit user up. + * + * An overwrite, the commit of a query in complete mode, publishes the complete result of its + * micro-batch: committing a replay of it again leaves the table as its first commit did, and runs + * the callbacks of an overwrite again, as the cleanup of the snapshot branch of a chain table + * needs. Every micro-batch of such a query is committed, as it always was. + */ + def commit( + write: StreamingWrite, + messages: Seq[CommitMessage], + table: FileStoreTable, + overwrite: Boolean): Unit = + synchronized { + require(committer != null, "The committer of the streaming query has not been created.") + val identifier = write.commitIdentifier + if (overwrite) { + committer.commit(identifier, messages.asJava) + } else if (lastCommitted.exists(identifier > _)) { + writeMarker(write, table) + committer.commit(identifier, messages.asJava) + } else { + commitFirstOfRun(write, messages, table) + } + lastCommitted = Some(lastCommitted.fold(identifier)(Math.max(_, identifier))) + } + + private def writeMarker(write: StreamingWrite, table: FileStoreTable): Unit = + marker.foreach( + _.write( + commitUser, + write.batchId, + Option(table.snapshotManager().latestSnapshotId()).map(_.longValue()).getOrElse(0L))) + + /** + * The first micro-batch of a run, which may be a replay. Whether its commit user already + * committed it is looked up here first, and only then is the lookup validated against snapshot + * expiration, so that the validation covers what the lookup saw. + * + * A micro-batch that was not committed is committed directly. One that was is not committed + * again, but is handed to the commit callbacks, through the committer's filtered commit, to retry + * what may have failed after its first commit, such as registering its partitions. It is handed + * over without its files: should expiration remove the first commit between this lookup and the + * one of the filtered commit, what gets published is an empty commit, not the batch a second + * time. + */ + private def commitFirstOfRun( + write: StreamingWrite, + messages: Seq[CommitMessage], + table: FileStoreTable): Unit = { + val identifier = write.commitIdentifier + val previousAttempt = marker.flatMap(_.latestSnapshotIdBefore(commitUser, write.batchId)) + writeMarker(write, table) + val latestOfUser = table.snapshotManager().latestSnapshotOfUser(commitUser) + if (latestOfUser.isPresent && latestOfUser.get().commitIdentifier() >= identifier) { Review Comment: Thanks, reproduced and fixed in 8effe4d. A replayed micro-batch no longer relies on the newest snapshot of its commit user: it looks for its identifier (or a later one) in every snapshot since the boundary its commit marker recorded. So the OVERWRITE that partition expiration of batch N-1 publishes after batch N can no longer hide N's commit; if the snapshot after that boundary has expired, the query still fails as before. New test `a replay is recognised after a late partition expiration of an earlier batch` builds your `1 APPEND -> 2 APPEND -> 1 OVERWRITE` sequence with the real `NormalPartitionExpire`: it wrote the row twice before, and passes now. A second one pins the boundary. Sink suites pass on Spark 3.2 / 3.4 / 3.5 / 4.0 / 4.1, and `paimon-spark-ut` on 3.5 apart from the lumina suites (no native library here). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
