This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20564-9a0974b8280c7ee2f15b0b4e400e4944f55069a8 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 843a0ab6510aca890e192a6cd3018dd722d683e5 Author: Shihuan Liu <[email protected]> AuthorDate: Wed Nov 11 02:58:13 2026 +0000 fix(flink): report stream read delays from the Source V2 reader and while it stalls (#20564) * fix(flink): report stream read delays from the Source V2 reader and while it stalls * fix(flink): finish split read delays once records are emitted and start them before opening On Source V2 the split reader finished the split's delay on the fetcher thread when it fetched the last batch, while earlier batches could still be queued for emission or blocked by backpressure, so splitLatestCommitDelay reported 0 while the split was still pending. It also started the delay only after the split was opened, hiding slow opens and recovery replays. Track active splits by id in FlinkStreamReadMetrics: the split reader starts a split before opening it, and HoodieSourceReader finishes it in onSplitFinished, once all of its records are emitted. splitLatestCommitDelay reports the oldest active split, so finishing an older split doesn't clear a newer one that the fetcher already started. --- .../hudi/metrics/FlinkStreamReadMetrics.java | 92 +++++++--- .../org/apache/hudi/source/StreamReadOperator.java | 3 +- .../hudi/source/reader/HoodieSourceReader.java | 27 ++- .../source/reader/HoodieSourceSplitReader.java | 17 +- .../hudi/metrics/TestFlinkStreamReadMetrics.java | 199 +++++++++++++++++++++ .../hudi/source/reader/TestHoodieSourceReader.java | 82 +++++++++ .../source/reader/TestHoodieSourceSplitReader.java | 114 ++++++++---- 7 files changed, 471 insertions(+), 63 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkStreamReadMetrics.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkStreamReadMetrics.java index 152cde237576..50400c8d7006 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkStreamReadMetrics.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkStreamReadMetrics.java @@ -19,76 +19,124 @@ package org.apache.hudi.metrics; import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator; +import org.apache.hudi.common.util.VisibleForTesting; import org.apache.hudi.source.StreamReadMonitoringFunction; import org.apache.hudi.source.StreamReadOperator; +import org.apache.hudi.source.reader.HoodieSourceSplitReader; import lombok.extern.slf4j.Slf4j; import org.apache.flink.metrics.MetricGroup; import java.text.ParseException; +import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Comparator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * Metrics for flink stream read. + * + * <p>The delays are computed when the gauges are read, so they keep growing while the reader stalls + * or discovery finds no new commits. The setters run on the reading threads and the gauges are read + * by the metric reporter thread. */ @Slf4j public class FlinkStreamReadMetrics extends HoodieFlinkMetrics { + private final Clock clock; + /** - * The last issued instant in streaming read. + * The last issued instant in streaming read, null until the first one is issued. * * @see StreamReadMonitoringFunction */ - private long issuedInstant; - - /** - * Duration between last issued instant and now. - */ - private long issuedInstantDelay; + private volatile Instant issuedInstant; /** - * Latest commit of current read split. + * Latest commit of the last split the reader started, null until the first one is started. * * @see StreamReadOperator + * @see HoodieSourceSplitReader */ - private long splitLatestCommit; + private volatile Instant splitLatestCommit; /** - * Duration between splitLatestCommit and now. + * Latest commits of the splits that were started and not yet finished, keyed by split id. On + * Source V2 a split starts on the fetcher thread and finishes on the task thread once all of its + * records are emitted, so the next split can start before the previous one finishes. */ - private long splitLatestCommitDelay; + private final Map<String, Instant> activeSplitCommits = new ConcurrentHashMap<>(); public FlinkStreamReadMetrics(MetricGroup metricGroup, String tableName) { + this(metricGroup, tableName, Clock.systemUTC()); + } + + @VisibleForTesting + FlinkStreamReadMetrics(MetricGroup metricGroup, String tableName, Clock clock) { super(metricGroup.addGroup("table", tableName)); + this.clock = clock; } @Override public void registerMetrics() { - metricGroup.gauge("issuedInstantDelay", () -> issuedInstantDelay); - metricGroup.gauge("issuedInstant", () -> issuedInstant); - metricGroup.gauge("splitLatestCommit", () -> splitLatestCommit); - metricGroup.gauge("splitLatestCommitDelay", () -> splitLatestCommitDelay); + metricGroup.gauge("issuedInstantDelay", this::getIssuedInstantDelay); + metricGroup.gauge("issuedInstant", () -> toEpochSecond(issuedInstant)); + metricGroup.gauge("splitLatestCommit", () -> toEpochSecond(splitLatestCommit)); + metricGroup.gauge("splitLatestCommitDelay", this::getSplitLatestCommitDelay); } public void setIssuedInstant(String issuedInstant) { try { - Instant instant = HoodieInstantTimeGenerator.parseDateFromInstantTime(issuedInstant).toInstant(); - this.issuedInstant = instant.getEpochSecond(); - this.issuedInstantDelay = Duration.between(instant, Instant.now()).getSeconds(); + this.issuedInstant = HoodieInstantTimeGenerator.parseDateFromInstantTime(issuedInstant).toInstant(); } catch (ParseException e) { log.warn("Invalid input issued instant: {}", issuedInstant); } } - public void setSplitLatestCommit(String splitLatestCommit) { + /** + * Records that the reader started a split whose latest commit is {@code splitLatestCommit}. Call it + * before opening the split, so that a slow open or recovery replay counts as reading. + */ + public void startSplit(String splitId, String splitLatestCommit) { try { - Instant instant = HoodieInstantTimeGenerator.parseDateFromInstantTime(splitLatestCommit).toInstant(); - this.splitLatestCommit = instant.getEpochSecond(); - this.splitLatestCommitDelay = Duration.between(instant, Instant.now()).getSeconds(); + Instant commit = HoodieInstantTimeGenerator.parseDateFromInstantTime(splitLatestCommit).toInstant(); + this.splitLatestCommit = commit; + activeSplitCommits.put(splitId, commit); } catch (ParseException e) { log.warn("Invalid input latest commit: {}", splitLatestCommit); } } + /** + * Records that all records of the split were emitted. Splits that were never started are ignored. + * Once no split is active, {@code splitLatestCommitDelay} reports 0 and {@code splitLatestCommit} + * keeps the last value. + */ + public void finishSplit(String splitId) { + activeSplitCommits.remove(splitId); + } + + private long getIssuedInstantDelay() { + return delaySeconds(issuedInstant); + } + + /** + * Returns the delay of the oldest active split, whose records are the stalest still to be emitted. + */ + private long getSplitLatestCommitDelay() { + return activeSplitCommits.values().stream() + .min(Comparator.naturalOrder()) + .map(this::delaySeconds) + .orElse(0L); + } + + private long delaySeconds(Instant instant) { + return instant == null ? 0L : Duration.between(instant, clock.instant()).getSeconds(); + } + + private static long toEpochSecond(Instant instant) { + return instant == null ? 0L : instant.getEpochSecond(); + } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/StreamReadOperator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/StreamReadOperator.java index 73028c59a728..7b20d5302d2b 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/StreamReadOperator.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/StreamReadOperator.java @@ -165,8 +165,8 @@ public class StreamReadOperator extends AbstractStreamOperator<RowData> // This log is important to indicate the consuming process, // there is only one log message for one data bucket. log.info("Processing input split : {}", split); + readMetrics.startSplit(String.valueOf(split.getSplitNumber()), split.getLatestCommit()); format.open(split); - readMetrics.setSplitLatestCommit(split.getLatestCommit()); } try { @@ -196,6 +196,7 @@ public class StreamReadOperator extends AbstractStreamOperator<RowData> } else { // close the input format format.close(); + readMetrics.finishSplit(String.valueOf(split.getSplitNumber())); // remove the split splits.poll(); break; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceReader.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceReader.java index 294039597fb7..14436fad5af4 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceReader.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceReader.java @@ -20,6 +20,8 @@ package org.apache.hudi.source.reader; import org.apache.hudi.common.function.SerializableSupplier; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.VisibleForTesting; +import org.apache.hudi.metrics.FlinkStreamReadMetrics; import org.apache.hudi.source.HoodieScanContext; import org.apache.hudi.source.reader.function.SplitReaderFunction; import org.apache.hudi.source.split.HoodieSourceSplit; @@ -41,6 +43,8 @@ import java.util.Map; */ public class HoodieSourceReader<T> extends SingleThreadMultiplexSourceReaderBase<HoodieRecordWithPosition<T>, T, HoodieSourceSplit, HoodieSourceSplit> { + private final FlinkStreamReadMetrics readerMetrics; + public HoodieSourceReader( String tableName, RecordEmitter<HoodieRecordWithPosition<T>, T, HoodieSourceSplit> recordEmitter, @@ -48,9 +52,27 @@ public class HoodieSourceReader<T> extends SourceReaderContext context, SerializableSupplier<SplitReaderFunction<T>> readerFunctionSupplier, SerializableComparator<HoodieSourceSplit> splitComparator) { - super(() -> new HoodieSourceSplitReader<>(tableName, context, readerFunctionSupplier, splitComparator, + this(createReaderMetrics(context, tableName), recordEmitter, scanContext, context, readerFunctionSupplier, splitComparator); + } + + @VisibleForTesting + HoodieSourceReader( + FlinkStreamReadMetrics readerMetrics, + RecordEmitter<HoodieRecordWithPosition<T>, T, HoodieSourceSplit> recordEmitter, + HoodieScanContext scanContext, + SourceReaderContext context, + SerializableSupplier<SplitReaderFunction<T>> readerFunctionSupplier, + SerializableComparator<HoodieSourceSplit> splitComparator) { + super(() -> new HoodieSourceSplitReader<>(readerMetrics, readerFunctionSupplier, splitComparator, scanContext.getLimit() == RecordLimiter.NO_LIMIT ? Option.empty() : Option.of(new RecordLimiter(scanContext.getLimit()))), recordEmitter, scanContext.getConf(), context); + this.readerMetrics = readerMetrics; + } + + private static FlinkStreamReadMetrics createReaderMetrics(SourceReaderContext context, String tableName) { + FlinkStreamReadMetrics readerMetrics = new FlinkStreamReadMetrics(context.metricGroup(), tableName); + readerMetrics.registerMetrics(); + return readerMetrics; } @Override @@ -64,6 +86,9 @@ public class HoodieSourceReader<T> extends @Override protected void onSplitFinished(Map<String, HoodieSourceSplit> finishedSplitIds) { + // Called on the task thread after all records of the splits are emitted, unlike the split + // reader, which finishes a split on the fetcher thread while its last batches are still queued. + finishedSplitIds.keySet().forEach(readerMetrics::finishSplit); requestSplit(new ArrayList<>(finishedSplitIds.keySet())); } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java index c12c4309df83..3d9c08bce8a8 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/reader/HoodieSourceSplitReader.java @@ -26,7 +26,6 @@ import org.apache.hudi.source.split.HoodieSourceSplit; import org.apache.hudi.source.split.SerializableComparator; import lombok.extern.slf4j.Slf4j; -import org.apache.flink.api.connector.source.SourceReaderContext; import org.apache.flink.connector.base.source.reader.RecordsBySplits; import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; @@ -72,9 +71,17 @@ public class HoodieSourceSplitReader<T> implements SplitReader<HoodieRecordWithP // Reset at the start of every fetch(), so it only ever means "a wakeUp() landed during THIS fetch". private volatile boolean wokenUp; + /** + * Creates a split reader. + * + * @param readerMetrics metrics of the source reader, already registered. Flink creates a new split + * reader each time it restarts an idle fetcher, so the instance is shared by + * all split readers of a source reader rather than registered per split reader. + * The split reader only starts splits; {@link HoodieSourceReader} finishes them + * once their records are emitted. + */ public HoodieSourceSplitReader( - String tableName, - SourceReaderContext context, + FlinkStreamReadMetrics readerMetrics, SerializableSupplier<SplitReaderFunction<T>> readerFunctionSupplier, SerializableComparator<HoodieSourceSplit> splitComparator, Option<RecordLimiter> recordLimiter) { @@ -84,8 +91,7 @@ public class HoodieSourceSplitReader<T> implements SplitReader<HoodieRecordWithP // fresh stateful cursor for each split reader so the old fetcher's close cannot affect the new one. this.readerFunction = readerFunctionSupplier.get(); this.recordLimiter = recordLimiter; - this.readerMetrics = new FlinkStreamReadMetrics(context.metricGroup(), tableName); - this.readerMetrics.registerMetrics(); + this.readerMetrics = readerMetrics; } @Override @@ -108,6 +114,7 @@ public class HoodieSourceSplitReader<T> implements SplitReader<HoodieRecordWithP return new RecordsBySplits<>(Collections.emptyMap(), Collections.emptySet()); } currentSplit = nextSplit; + readerMetrics.startSplit(currentSplit.splitId(), currentSplit.getLatestCommit()); readerFunction.open(currentSplit); } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkStreamReadMetrics.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkStreamReadMetrics.java new file mode 100644 index 000000000000..4e219ea0625e --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkStreamReadMetrics.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.metrics; + +import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator; + +import org.apache.flink.metrics.Gauge; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.text.ParseException; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests for {@link FlinkStreamReadMetrics}. + */ +class TestFlinkStreamReadMetrics { + + private static final String INSTANT = "20261109100000000"; + private static final String NEWER_INSTANT = "20261109110000000"; + + private CapturingMetricGroup metricGroup; + private MutableClock clock; + private FlinkStreamReadMetrics metrics; + + @BeforeEach + void setUp() throws ParseException { + metricGroup = new CapturingMetricGroup(); + clock = new MutableClock(toInstant(INSTANT)); + metrics = new FlinkStreamReadMetrics(metricGroup, "test_table", clock); + metrics.registerMetrics(); + } + + @Test + void testGaugesReportZeroBeforeAnyValueIsSet() { + clock.advance(Duration.ofHours(1)); + assertEquals(0L, gauge("issuedInstant")); + assertEquals(0L, gauge("issuedInstantDelay")); + assertEquals(0L, gauge("splitLatestCommit")); + assertEquals(0L, gauge("splitLatestCommitDelay")); + } + + @Test + void testIssuedInstantDelayGrowsUntilANewInstantIsIssued() throws ParseException { + metrics.setIssuedInstant(INSTANT); + assertEquals(toInstant(INSTANT).getEpochSecond(), gauge("issuedInstant")); + assertEquals(0L, gauge("issuedInstantDelay")); + + clock.advance(Duration.ofMinutes(90)); + assertEquals(5400L, gauge("issuedInstantDelay")); + + metrics.setIssuedInstant(NEWER_INSTANT); + assertEquals(toInstant(NEWER_INSTANT).getEpochSecond(), gauge("issuedInstant")); + assertEquals(1800L, gauge("issuedInstantDelay")); + } + + @Test + void testSplitLatestCommitDelayGrowsWhileReadingASplit() throws ParseException { + metrics.startSplit("split-1", INSTANT); + assertEquals(toInstant(INSTANT).getEpochSecond(), gauge("splitLatestCommit")); + + clock.advance(Duration.ofMinutes(10)); + assertEquals(600L, gauge("splitLatestCommitDelay")); + clock.advance(Duration.ofMinutes(10)); + assertEquals(1200L, gauge("splitLatestCommitDelay")); + } + + @Test + void testSplitLatestCommitDelayIsZeroWithoutAnActiveSplit() throws ParseException { + metrics.startSplit("split-1", INSTANT); + clock.advance(Duration.ofMinutes(10)); + metrics.finishSplit("split-1"); + + assertEquals(0L, gauge("splitLatestCommitDelay")); + assertEquals(toInstant(INSTANT).getEpochSecond(), gauge("splitLatestCommit"), + "the last split's commit is still reported"); + + metrics.startSplit("split-2", INSTANT); + assertEquals(600L, gauge("splitLatestCommitDelay")); + } + + @Test + void testSplitLatestCommitDelayReportsTheOldestActiveSplit() throws ParseException { + metrics.startSplit("split-1", INSTANT); + metrics.startSplit("split-2", NEWER_INSTANT); + clock.advance(Duration.ofHours(2)); + + assertEquals(7200L, gauge("splitLatestCommitDelay")); + assertEquals(toInstant(NEWER_INSTANT).getEpochSecond(), gauge("splitLatestCommit")); + } + + @Test + void testFinishingAnOlderSplitKeepsTheNewerSplitActive() { + metrics.startSplit("split-1", INSTANT); + metrics.startSplit("split-2", NEWER_INSTANT); + clock.advance(Duration.ofHours(2)); + + metrics.finishSplit("split-1"); + + assertEquals(3600L, gauge("splitLatestCommitDelay")); + } + + @Test + void testFinishingASplitThatWasNotStartedIsIgnored() { + metrics.startSplit("split-1", INSTANT); + clock.advance(Duration.ofMinutes(10)); + + metrics.finishSplit("split-2"); + + assertEquals(600L, gauge("splitLatestCommitDelay")); + } + + @Test + void testInvalidInstantKeepsThePreviousValue() throws ParseException { + metrics.setIssuedInstant(INSTANT); + metrics.setIssuedInstant("not-an-instant"); + assertEquals(toInstant(INSTANT).getEpochSecond(), gauge("issuedInstant")); + + metrics.startSplit("split-1", "not-an-instant"); + assertEquals(0L, gauge("splitLatestCommit")); + assertEquals(0L, gauge("splitLatestCommitDelay")); + } + + private long gauge(String name) { + return (Long) metricGroup.gauges.get(name).getValue(); + } + + private static Instant toInstant(String instantTime) throws ParseException { + return HoodieInstantTimeGenerator.parseDateFromInstantTime(instantTime).toInstant(); + } + + private static class CapturingMetricGroup extends UnregisteredMetricsGroup { + private final Map<String, Gauge<?>> gauges = new HashMap<>(); + + @Override + public MetricGroup addGroup(String key, String value) { + return this; + } + + @Override + public <T, G extends Gauge<T>> G gauge(String name, G gauge) { + gauges.put(name, gauge); + return gauge; + } + } + + private static class MutableClock extends Clock { + private Instant now; + + private MutableClock(Instant now) { + this.now = now; + } + + private void advance(Duration duration) { + now = now.plus(duration); + } + + @Override + public Instant instant() { + return now; + } + + @Override + public ZoneId getZone() { + return ZoneOffset.UTC; + } + + @Override + public Clock withZone(ZoneId zone) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceReader.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceReader.java new file mode 100644 index 000000000000..012acc82502f --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceReader.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.source.reader; + +import org.apache.hudi.common.util.Option; +import org.apache.hudi.metrics.FlinkStreamReadMetrics; +import org.apache.hudi.source.HoodieScanContext; +import org.apache.hudi.source.reader.function.SplitReaderFunction; +import org.apache.hudi.source.split.HoodieSourceSplit; +import org.apache.hudi.source.split.SplitRequestEvent; + +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.base.source.reader.RecordEmitter; +import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Test cases for {@link HoodieSourceReader}. + */ +public class TestHoodieSourceReader { + + @Test + @SuppressWarnings("unchecked") + public void testMetricsFinishSplitsOnceTheirRecordsAreEmitted() throws Exception { + FlinkStreamReadMetrics metrics = Mockito.mock(FlinkStreamReadMetrics.class); + SourceReaderContext context = Mockito.mock(SourceReaderContext.class); + when(context.metricGroup()).thenReturn(UnregisteredMetricsGroup.createSourceReaderMetricGroup()); + HoodieScanContext scanContext = HoodieScanContext.builder() + .conf(new Configuration()) + .limit(RecordLimiter.NO_LIMIT) + .build(); + HoodieSourceReader<String> reader = new HoodieSourceReader<>( + metrics, + Mockito.mock(RecordEmitter.class), + scanContext, + context, + () -> Mockito.mock(SplitReaderFunction.class), + null); + Map<String, HoodieSourceSplit> finishedSplits = new LinkedHashMap<>(); + finishedSplits.put("split-1", createSplit(1)); + finishedSplits.put("split-2", createSplit(2)); + + reader.onSplitFinished(finishedSplits); + + verify(metrics).finishSplit("split-1"); + verify(metrics).finishSplit("split-2"); + verify(context).sendSourceEventToCoordinator(any(SplitRequestEvent.class)); + reader.close(); + } + + private static HoodieSourceSplit createSplit(int splitNum) { + return new HoodieSourceSplit(splitNum, "base-path-" + splitNum, Option.of(Collections.emptyList()), + "/test/table", "/test/partition", "read_optimized", "20261109100000000", "file" + splitNum, + Option.empty()); + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java index 11e2cd940ea7..915b3f2544fc 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/reader/TestHoodieSourceSplitReader.java @@ -20,11 +20,11 @@ package org.apache.hudi.source.reader; import org.apache.hudi.common.function.SerializableSupplier; import org.apache.hudi.common.util.Option; +import org.apache.hudi.metrics.FlinkStreamReadMetrics; import org.apache.hudi.source.reader.function.SplitReaderFunction; import org.apache.hudi.source.split.HoodieSourceSplit; import org.apache.hudi.source.split.SerializableComparator; -import org.apache.flink.api.connector.source.SourceReaderContext; import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; import org.apache.flink.connector.base.source.reader.splitreader.SplitsAddition; import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; @@ -50,26 +50,28 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; /** * Test cases for {@link HoodieSourceSplitReader}. */ public class TestHoodieSourceSplitReader { private static final String TABLE_NAME = "test_table"; - private SourceReaderContext readerContext; + private FlinkStreamReadMetrics readerMetrics; @BeforeEach public void setUp() { - readerContext = Mockito.mock(SourceReaderContext.class); - doReturn(UnregisteredMetricsGroup.createSourceReaderMetricGroup()).when(readerContext).metricGroup(); + readerMetrics = new FlinkStreamReadMetrics(UnregisteredMetricsGroup.createSourceReaderMetricGroup(), TABLE_NAME); + readerMetrics.registerMetrics(); } @Test public void testFetchWithNoSplits() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); RecordsWithSplitIds<HoodieRecordWithPosition<String>> result = reader.fetch(); @@ -83,7 +85,7 @@ public class TestHoodieSourceSplitReader { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); SplitsAddition<HoodieSourceSplit> splitsChange = new SplitsAddition<>(Collections.singletonList(split)); @@ -101,7 +103,7 @@ public class TestHoodieSourceSplitReader { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -129,7 +131,7 @@ public class TestHoodieSourceSplitReader { (s1, s2) -> s2.getFileId().compareTo(s1.getFileId()); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, comparator, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, comparator, Option.empty()); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -152,7 +154,7 @@ public class TestHoodieSourceSplitReader { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); // First batch HoodieSourceSplit split1 = createTestSplit(1, "file1"); @@ -174,7 +176,7 @@ public class TestHoodieSourceSplitReader { public void testClose() throws Exception { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); @@ -193,7 +195,7 @@ public class TestHoodieSourceSplitReader { public void testWakeUp() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); // wakeUp() now sets a flag but must not throw, and the flag is reset at the top of each fetch() // so a wakeUp with no in-flight drain leaves the next fetch() unaffected. @@ -215,7 +217,7 @@ public class TestHoodieSourceSplitReader { List<String> testData = Arrays.asList("r1", "r2", "r3", "r4", "r5"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); boolean[] fired = {false}; readerFunction.setDrainProbe((buffered, hasNext) -> { if (buffered == 1 && !fired[0]) { @@ -262,7 +264,7 @@ public class TestHoodieSourceSplitReader { List<String> testData = Arrays.asList("r1", "r2", "r3"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); boolean[] fired = {false}; readerFunction.setDrainProbe((buffered, hasNext) -> { // Wake at the start of the drain (count 0) while data is still available. @@ -294,7 +296,7 @@ public class TestHoodieSourceSplitReader { List<String> testData = Arrays.asList("r1", "r2"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); boolean[] fired = {false}; readerFunction.setDrainProbe((buffered, hasNext) -> { // Wake only at the start of a drain that finds the cursor already exhausted. @@ -342,7 +344,7 @@ public class TestHoodieSourceSplitReader { } }; HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(wakingLimiter)); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.of(wakingLimiter)); holder.set(reader); HoodieSourceSplit split = createTestSplit(1, "file1"); @@ -366,7 +368,7 @@ public class TestHoodieSourceSplitReader { List<String> testData = Arrays.asList("r1", "r2", "r3", "r4", "r5"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(3L))); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.of(new RecordLimiter(3L))); boolean[] fired = {false}; readerFunction.setDrainProbe((buffered, hasNext) -> { if (buffered == 1 && !fired[0]) { @@ -399,7 +401,7 @@ public class TestHoodieSourceSplitReader { List<String> testData = Arrays.asList("r1", "r2", "r3"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); boolean[] fired = {false}; readerFunction.setDrainProbe((buffered, hasNext) -> { if (buffered == 1 && !fired[0]) { @@ -425,7 +427,7 @@ public class TestHoodieSourceSplitReader { public void testPauseOrResumeSplits() { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -443,7 +445,7 @@ public class TestHoodieSourceSplitReader { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); @@ -454,11 +456,50 @@ public class TestHoodieSourceSplitReader { assertEquals(split, readerFunction.getLastReadSplit()); } + @Test + public void testMetricsStartTheSplitBeforeOpeningIt() throws IOException { + FlinkStreamReadMetrics metrics = Mockito.mock(FlinkStreamReadMetrics.class); + HoodieSourceSplit split = createSplitWithCommit("20261109100000000"); + // A slow open or recovery replay must already count as reading the split. + TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(Arrays.asList("r1", "r2")) { + @Override + public void open(HoodieSourceSplit openedSplit) { + verify(metrics).startSplit(split.splitId(), "20261109100000000"); + super.open(openedSplit); + } + }; + HoodieSourceSplitReader<String> reader = + new HoodieSourceSplitReader<>(metrics, () -> readerFunction, null, Option.empty()); + reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); + + reader.fetch(); + + assertEquals(1, readerFunction.getOpenCount()); + verify(metrics, times(1)).startSplit(Mockito.anyString(), Mockito.anyString()); + } + + @Test + public void testMetricsSplitIsNotFinishedWhenItsLastBatchIsFetched() throws IOException { + FlinkStreamReadMetrics metrics = Mockito.mock(FlinkStreamReadMetrics.class); + TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(Arrays.asList("r1", "r2")); + HoodieSourceSplitReader<String> reader = + new HoodieSourceSplitReader<>(metrics, () -> readerFunction, null, Option.empty()); + HoodieSourceSplit split = createSplitWithCommit("20261109100000000"); + reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); + reader.fetch(); + + RecordsWithSplitIds<HoodieRecordWithPosition<String>> finishBatch = reader.fetch(); + + // Earlier batches may still be queued for emission, so the source reader finishes the split. + assertTrue(finishBatch.finishedSplits().contains(split.splitId())); + verify(metrics, never()).finishSplit(Mockito.anyString()); + } + @Test public void testReaderFunctionClosedOnReaderClose() throws Exception { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); reader.close(); @@ -475,9 +516,9 @@ public class TestHoodieSourceSplitReader { }; HoodieSourceSplitReader<String> firstReader = new HoodieSourceSplitReader<>( - TABLE_NAME, readerContext, readerFunctionSupplier, null, Option.empty()); + readerMetrics, readerFunctionSupplier, null, Option.empty()); HoodieSourceSplitReader<String> secondReader = new HoodieSourceSplitReader<>( - TABLE_NAME, readerContext, readerFunctionSupplier, null, Option.empty()); + readerMetrics, readerFunctionSupplier, null, Option.empty()); assertEquals(2, readerFunctions.size()); firstReader.close(); @@ -493,7 +534,7 @@ public class TestHoodieSourceSplitReader { public void testFetchEmptyResultWhenNoSplitsAdded() throws IOException { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); RecordsWithSplitIds<HoodieRecordWithPosition<String>> result = reader.fetch(); @@ -509,7 +550,7 @@ public class TestHoodieSourceSplitReader { // No comparator - should preserve insertion order HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split3 = createTestSplit(3, "file3"); HoodieSourceSplit split1 = createTestSplit(1, "file1"); @@ -531,7 +572,7 @@ public class TestHoodieSourceSplitReader { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -553,7 +594,7 @@ public class TestHoodieSourceSplitReader { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(2L))); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.of(new RecordLimiter(2L))); HoodieSourceSplit split = createTestSplit(1, "file1"); reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); @@ -581,7 +622,7 @@ public class TestHoodieSourceSplitReader { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(2L))); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.of(new RecordLimiter(2L))); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -609,7 +650,7 @@ public class TestHoodieSourceSplitReader { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(Arrays.asList("r1", "r2")); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(0L))); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.of(new RecordLimiter(0L))); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -632,7 +673,7 @@ public class TestHoodieSourceSplitReader { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(3L))); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.of(new RecordLimiter(3L))); HoodieSourceSplit split = createTestSplit(1, "file1"); reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); @@ -658,7 +699,7 @@ public class TestHoodieSourceSplitReader { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.of(new RecordLimiter(5L))); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.of(new RecordLimiter(5L))); HoodieSourceSplit split1 = createTestSplit(1, "file1"); HoodieSourceSplit split2 = createTestSplit(2, "file2"); @@ -691,7 +732,7 @@ public class TestHoodieSourceSplitReader { TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); @@ -715,7 +756,7 @@ public class TestHoodieSourceSplitReader { List<String> testData = IntStream.range(0, n).mapToObj(i -> "r" + i).collect(Collectors.toList()); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(split))); @@ -758,7 +799,7 @@ public class TestHoodieSourceSplitReader { List<String> testData = Arrays.asList("r1", "r2", "r3", "r4", "r5"); TestSplitReaderFunction readerFunction = new TestSplitReaderFunction(testData); HoodieSourceSplitReader<String> reader = - new HoodieSourceSplitReader<>(TABLE_NAME, readerContext, () -> readerFunction, null, Option.empty()); + new HoodieSourceSplitReader<>(readerMetrics, () -> readerFunction, null, Option.empty()); HoodieSourceSplit split = createTestSplit(1, "file1"); split.updatePosition(0, 2L); // 2 records already consumed before recovery @@ -819,6 +860,11 @@ public class TestHoodieSourceSplitReader { ); } + private HoodieSourceSplit createSplitWithCommit(String latestCommit) { + return new HoodieSourceSplit(1, "base-path-1", Option.of(Collections.emptyList()), + "/test/table", "/test/partition", "read_optimized", latestCommit, "file1", Option.empty()); + } + /** * Test implementation of the stateful {@link SplitReaderFunction} cursor contract: {@code open} * materializes the split's data and honors the consumed-offset skip, {@code readBatch} drains a
