This is an automated email from the ASF dual-hosted git repository.
nzw921rx pushed a commit to branch 3.1.0-release
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/3.1.0-release by this push:
new d38c438e37 [Improve][API] Reduce queue handoff overhead in
MultiTableSinkWriter (#13196)
d38c438e37 is described below
commit d38c438e37003318aaea3daba0812b8b0b28bbcc
Author: zhiweiniu <[email protected]>
AuthorDate: Mon Nov 9 23:09:16 2026 +0800
[Improve][API] Reduce queue handoff overhead in MultiTableSinkWriter
(#13196)
(cherry picked from commit e1fe5c5ef22c71a483dffad3f34fe202774a2d61)
---
.../sink/multitablesink/MultiTableSinkWriter.java | 3 +-
.../multitablesink/MultiTableWriterRunnable.java | 35 ++++++++++++++++++++++
2 files changed, 37 insertions(+), 1 deletion(-)
diff --git
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
index 590c35bf9a..6037894acd 100644
---
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
+++
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
@@ -250,7 +250,8 @@ public class MultiTableSinkWriter
failurePolicy.continueOtherTables(),
this::handleRuntimeTableFailure,
this.tableRetryTimes,
- this.tableRetryIntervalSeconds);
+ this.tableRetryIntervalSeconds,
+ queueSize > 1);
runnable.add(r);
}
log.info("init multi table sink writer, queue size: {}", queueSize);
diff --git
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableWriterRunnable.java
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableWriterRunnable.java
index 906cd77297..014525aa92 100644
---
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableWriterRunnable.java
+++
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableWriterRunnable.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.LockSupport;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
@@ -39,6 +40,9 @@ import java.util.function.Consumer;
@Slf4j
public class MultiTableWriterRunnable implements Runnable {
+ private static final long HANDOFF_COALESCE_NANOS =
TimeUnit.MICROSECONDS.toNanos(50);
+ private static final long HANDOFF_BURST_NANOS =
TimeUnit.MICROSECONDS.toNanos(200);
+
/** Writers that belong to this queue, keyed by logical source-table
identifier. */
private final Map<String, SinkWriter<SeaTunnelRow, ?, ?>> tableIdWriterMap;
/** Ordered requests for this queue: data rows or schema-change barrier
markers. */
@@ -53,6 +57,8 @@ public class MultiTableWriterRunnable implements Runnable {
private final int tableRetryTimes;
/** Sleep interval between write retries for continue-other-tables mode. */
private final int tableRetryIntervalSeconds;
+
+ private final boolean coalesceHandoffs;
/** First fatal worker failure surfaced back to the coordinator. */
private volatile Throwable throwable;
/** Table currently being written, used in fail-fast diagnostics. */
@@ -89,6 +95,24 @@ public class MultiTableWriterRunnable implements Runnable {
BiConsumer<String, Throwable> failureHandler,
int tableRetryTimes,
int tableRetryIntervalSeconds) {
+ this(
+ tableIdWriterMap,
+ queue,
+ continueOnTableFailure,
+ failureHandler,
+ tableRetryTimes,
+ tableRetryIntervalSeconds,
+ false);
+ }
+
+ MultiTableWriterRunnable(
+ Map<String, SinkWriter<SeaTunnelRow, ?, ?>> tableIdWriterMap,
+ BlockingQueue<QueueElement> queue,
+ boolean continueOnTableFailure,
+ BiConsumer<String, Throwable> failureHandler,
+ int tableRetryTimes,
+ int tableRetryIntervalSeconds,
+ boolean coalesceHandoffs) {
this.tableIdWriterMap = tableIdWriterMap;
this.queue = queue;
this.allowSingleWriterFallback = tableIdWriterMap.size() == 1;
@@ -96,6 +120,7 @@ public class MultiTableWriterRunnable implements Runnable {
this.failureHandler = failureHandler;
this.tableRetryTimes = Math.max(0, tableRetryTimes);
this.tableRetryIntervalSeconds = Math.max(0,
tableRetryIntervalSeconds);
+ this.coalesceHandoffs = coalesceHandoffs;
}
public void setRowErrorHandler(MultiTableRowErrorHandler rowErrorHandler) {
@@ -108,6 +133,7 @@ public class MultiTableWriterRunnable implements Runnable {
@Override
public void run() {
+ long lastEmptyQueueNanos = System.nanoTime();
while (true) {
QueueElement queueElement = null;
TableFailure tableFailure = null;
@@ -159,6 +185,15 @@ public class MultiTableWriterRunnable implements Runnable {
pendingRowRequests.decrementAndGet();
}
}
+ if (coalesceHandoffs && queueElement.isRowRequest() &&
queue.isEmpty()) {
+ long now = System.nanoTime();
+ if (now - lastEmptyQueueNanos <= HANDOFF_BURST_NANOS) {
+ // A frequently empty FIFO can collect several rows during
this brief pause.
+ // Every accepted row remains visible to the existing
drain checks.
+ LockSupport.parkNanos(HANDOFF_COALESCE_NANOS);
+ }
+ lastEmptyQueueNanos = now;
+ }
}
}