This is an automated email from the ASF dual-hosted git repository.
nzw921rx pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 349ec2e577 [Test][Benchmark] Add MultiTableSink timer flush benchmark
(#13210)
349ec2e577 is described below
commit 349ec2e5774923ddd92b75c46c80d2ae9eacfd59
Author: zhiweiniu <[email protected]>
AuthorDate: Thu Nov 12 21:19:34 2026 +0800
[Test][Benchmark] Add MultiTableSink timer flush benchmark (#13210)
---
.github/workflows/benchmarks.yml | 1 +
seatunnel-benchmarks/README.md | 9 +-
.../benchmark/MultiTableSinkBenchmark.java | 73 +++++++++-
.../MultiTableSinkFixture.java | 2 +-
.../MultiTableTimerFlushFixture.java | 153 +++++++++++++++++++++
.../MultiTableSinkFixtureTest.java | 2 +-
6 files changed, 234 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 4a9d0e3b2e..6f55dee946 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -36,6 +36,7 @@ on:
- 'benchmarks_core'
- 'SeaTunnelRowBenchmark'
- 'IntermediateQueueBenchmark'
+ - 'MultiTableSinkBenchmark'
- 'DebeziumJsonFormatBenchmark'
- 'ProtoStuffSerializerBenchmark'
- 'SeaTunnelPipelineBenchmark'
diff --git a/seatunnel-benchmarks/README.md b/seatunnel-benchmarks/README.md
index 3016b44371..25834cb938 100644
--- a/seatunnel-benchmarks/README.md
+++ b/seatunnel-benchmarks/README.md
@@ -50,12 +50,19 @@ java -jar seatunnel-benchmarks/target/benchmarks.jar
ProtoStuffSerializerBenchma
Run the `MultiTableSink` record handoff benchmark without starting an engine
or database:
```bash
-java -jar seatunnel-benchmarks/target/benchmarks.jar MultiTableSinkBenchmark
-p replicaNum=1,5,15
+java -jar seatunnel-benchmarks/target/benchmarks.jar
'MultiTableSinkBenchmark.recordHandoff$' \
+ -p replicaNum=1,5,15
```
The score measures bounded record handoff throughput with per-writer sequence
checks; final
draining and count verification run outside the timed iteration.
+Run the `MultiTableSink` timer flush benchmark.
+
+```bash
+java -jar seatunnel-benchmarks/target/benchmarks.jar
'MultiTableSinkBenchmark.timerFlush$'
+```
+
Write JSON results:
```bash
diff --git
a/seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/MultiTableSinkBenchmark.java
b/seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/MultiTableSinkBenchmark.java
index f7919830f5..074baed85c 100644
---
a/seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/MultiTableSinkBenchmark.java
+++
b/seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/MultiTableSinkBenchmark.java
@@ -17,10 +17,14 @@
package org.apache.seatunnel.benchmark;
-import org.apache.seatunnel.benchmark.sink.MultiTableSinkFixture;
+import org.apache.seatunnel.benchmark.multitablesink.MultiTableSinkFixture;
+import
org.apache.seatunnel.benchmark.multitablesink.MultiTableTimerFlushFixture;
import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
@@ -34,9 +38,19 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
import java.util.concurrent.TimeUnit;
-/** Measures bounded, asynchronous MultiTableSink handoff without starting an
engine or database. */
-@OutputTimeUnit(TimeUnit.SECONDS)
+/** Measures MultiTableSink record handoff and timer flush using in-memory
writers. */
@Threads(1)
+@Fork(
+ value = 3,
+ jvmArgsAppend = {
+ "-Xms4g",
+ "-Xmx4g",
+ "-XX:+UseG1GC",
+ "-XX:+AlwaysPreTouch",
+ "-XX:+DisableExplicitGC",
+ "-XX:ActiveProcessorCount=4",
+ "-Djava.net.preferIPv4Stack=true"
+ })
public class MultiTableSinkBenchmark extends BenchmarkBase {
public static void main(String[] args) throws RunnerException {
@@ -48,10 +62,18 @@ public class MultiTableSinkBenchmark extends BenchmarkBase {
}
@Benchmark
+ @OutputTimeUnit(TimeUnit.SECONDS)
public long recordHandoff(SinkState state) throws Exception {
return state.fixture.publish();
}
+ @Benchmark
+ @BenchmarkMode(Mode.AverageTime)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
+ public void timerFlush(TimerFlushSinkState state) throws Exception {
+ state.fixture.flush();
+ }
+
@State(Scope.Thread)
public static class SinkState {
@@ -100,4 +122,49 @@ public class MultiTableSinkBenchmark extends BenchmarkBase
{
}
}
}
+
+ @State(Scope.Thread)
+ public static class TimerFlushSinkState {
+
+ @Param({"1", "5", "10"})
+ private int flushQueueCount;
+
+ @Param({"100", "500", "1000"})
+ private long flushIntervalMillis;
+
+ @Param({"5"})
+ private long writerFlushMillis;
+
+ private MultiTableTimerFlushFixture fixture;
+ private long intervalNanos;
+ private long nextFlushNanos;
+
+ @Setup(Level.Trial)
+ public void setUp() throws Exception {
+ if (flushIntervalMillis <= 0) {
+ throw new IllegalArgumentException("flushIntervalMillis must
be positive");
+ }
+ fixture = new MultiTableTimerFlushFixture(flushQueueCount,
writerFlushMillis);
+ intervalNanos = TimeUnit.MILLISECONDS.toNanos(flushIntervalMillis);
+ nextFlushNanos = System.nanoTime();
+ }
+
+ @Setup(Level.Invocation)
+ public void awaitFlushSignal() throws InterruptedException {
+ TimeUnit.NANOSECONDS.sleep(nextFlushNanos - System.nanoTime());
+ nextFlushNanos = System.nanoTime() + intervalNanos;
+ }
+
+ @TearDown(Level.Invocation)
+ public void verifyRound() {
+ fixture.verifyRound();
+ }
+
+ @TearDown(Level.Trial)
+ public void tearDown() throws Exception {
+ if (fixture != null) {
+ fixture.close();
+ }
+ }
+ }
}
diff --git
a/seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/sink/MultiTableSinkFixture.java
b/seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/multitablesink/MultiTableSinkFixture.java
similarity index 99%
rename from
seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/sink/MultiTableSinkFixture.java
rename to
seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/multitablesink/MultiTableSinkFixture.java
index ae6d3e6be8..fd4f5b3454 100644
---
a/seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/sink/MultiTableSinkFixture.java
+++
b/seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/multitablesink/MultiTableSinkFixture.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.seatunnel.benchmark.sink;
+package org.apache.seatunnel.benchmark.multitablesink;
import org.apache.seatunnel.api.common.metrics.MetricsContext;
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
diff --git
a/seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/multitablesink/MultiTableTimerFlushFixture.java
b/seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/multitablesink/MultiTableTimerFlushFixture.java
new file mode 100644
index 0000000000..7e099a2871
--- /dev/null
+++
b/seatunnel-benchmarks/src/main/java/org/apache/seatunnel/benchmark/multitablesink/MultiTableTimerFlushFixture.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.benchmark.multitablesink;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.options.SinkConnectorCommonOptions;
+import org.apache.seatunnel.api.sink.DefaultSinkWriterContext;
+import org.apache.seatunnel.api.sink.SeaTunnelSink;
+import org.apache.seatunnel.api.sink.SinkWriter;
+import org.apache.seatunnel.api.sink.SupportMultiTableSink;
+import org.apache.seatunnel.api.sink.SupportMultiTableSinkWriter;
+import org.apache.seatunnel.api.sink.multitablesink.MultiTableSink;
+import org.apache.seatunnel.api.sink.multitablesink.SinkContextProxy;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.factory.MultiTableFactoryContext;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.common.utils.function.RunnableWithException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+/** Exercises the production timer flush callback with two simulated buffered
batches per queue. */
+public final class MultiTableTimerFlushFixture implements AutoCloseable {
+
+ private final List<FlushWriter> writers = new ArrayList<>();
+ private final SinkWriter<SeaTunnelRow, ?, ?> coordinator;
+ private final RunnableWithException flushAction;
+ private long rounds;
+
+ public MultiTableTimerFlushFixture(int queueCount, long writerFlushMillis)
throws IOException {
+ if (writerFlushMillis < 0) {
+ throw new IllegalArgumentException("writerFlushMillis must be
non-negative");
+ }
+ FlushSink childSink = new FlushSink(writerFlushMillis);
+ Map<TablePath, SeaTunnelSink> sinks =
+ Map.of(
+ TablePath.of("benchmark", "table_0"), childSink,
+ TablePath.of("benchmark", "table_1"), childSink);
+ ReadonlyConfig options =
+ ReadonlyConfig.fromMap(
+ Map.of(
+
SinkConnectorCommonOptions.MULTI_TABLE_SINK_REPLICA.key(),
+ queueCount));
+ MultiTableSink sink =
+ new MultiTableSink(
+ new MultiTableFactoryContext(options,
getClass().getClassLoader(), sinks));
+ SinkWriter.Context context = new SinkContextProxy(0, 1, new
DefaultSinkWriterContext(0, 1));
+ coordinator = sink.createWriter(context);
+ flushAction = context.getFlushAction();
+ if (flushAction == null) {
+ coordinator.close();
+ throw new IllegalStateException("MultiTableSink did not register a
timer flush action");
+ }
+ }
+
+ public void flush() throws Exception {
+ flushAction.run();
+ }
+
+ /** Confirms that the callback waited for one flush from every child
writer. */
+ public void verifyRound() {
+ rounds++;
+ for (int i = 0; i < writers.size(); i++) {
+ long flushedBatches = writers.get(i).flushedBatches;
+ if (flushedBatches != rounds) {
+ throw new IllegalStateException(
+ String.format(
+ "Writer %d flushed %d batches after %d timer
rounds",
+ i, flushedBatches, rounds));
+ }
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ coordinator.close();
+ }
+
+ private final class FlushSink
+ implements SeaTunnelSink<SeaTunnelRow, Void, Void, Void>,
SupportMultiTableSink {
+
+ private final long writerFlushMillis;
+
+ private FlushSink(long writerFlushMillis) {
+ this.writerFlushMillis = writerFlushMillis;
+ }
+
+ @Override
+ public SinkWriter<SeaTunnelRow, Void, Void>
createWriter(SinkWriter.Context context) {
+ FlushWriter writer = new FlushWriter(writerFlushMillis);
+ context.registerFlushAction(writer::flush);
+ writers.add(writer);
+ return writer;
+ }
+
+ @Override
+ public String getPluginName() {
+ return "MultiTableTimerFlushBenchmarkSink";
+ }
+ }
+
+ private static final class FlushWriter
+ implements SinkWriter<SeaTunnelRow, Void, Void>,
SupportMultiTableSinkWriter<Void> {
+
+ private final long writerFlushMillis;
+ private long flushedBatches;
+
+ private FlushWriter(long writerFlushMillis) {
+ this.writerFlushMillis = writerFlushMillis;
+ }
+
+ private void flush() throws InterruptedException {
+ TimeUnit.MILLISECONDS.sleep(writerFlushMillis);
+ flushedBatches++;
+ }
+
+ @Override
+ public void write(SeaTunnelRow row) {
+ throw new UnsupportedOperationException(
+ "Only timer flush is supported by this fixture");
+ }
+
+ @Override
+ public Optional<Void> prepareCommit() {
+ return Optional.empty();
+ }
+
+ @Override
+ public void abortPrepare() {}
+
+ @Override
+ public void close() {}
+ }
+}
diff --git
a/seatunnel-benchmarks/src/test/java/org/apache/seatunnel/benchmark/sink/MultiTableSinkFixtureTest.java
b/seatunnel-benchmarks/src/test/java/org/apache/seatunnel/benchmark/multitablesink/MultiTableSinkFixtureTest.java
similarity index 99%
rename from
seatunnel-benchmarks/src/test/java/org/apache/seatunnel/benchmark/sink/MultiTableSinkFixtureTest.java
rename to
seatunnel-benchmarks/src/test/java/org/apache/seatunnel/benchmark/multitablesink/MultiTableSinkFixtureTest.java
index d91dc03247..28ff23645f 100644
---
a/seatunnel-benchmarks/src/test/java/org/apache/seatunnel/benchmark/sink/MultiTableSinkFixtureTest.java
+++
b/seatunnel-benchmarks/src/test/java/org/apache/seatunnel/benchmark/multitablesink/MultiTableSinkFixtureTest.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.seatunnel.benchmark.sink;
+package org.apache.seatunnel.benchmark.multitablesink;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;