This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 4b80fa56a99 Fix subscription prefetch retries under delivery
backpressure (#19137)
4b80fa56a99 is described below
commit 4b80fa56a999baf9de625f10b392bf2f2bcaa880
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 5 15:01:05 2026 +0800
Fix subscription prefetch retries under delivery backpressure (#19137)
---
.../consensus/ConsensusPrefetchingQueue.java | 7 +-
...nsensusPrefetchingQueueWalBackpressureTest.java | 195 +++++++++++++++++++++
2 files changed, 200 insertions(+), 2 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
index 5bcfd58d52d..765202b74f2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
@@ -2177,7 +2177,7 @@ public class ConsensusPrefetchingQueue {
}
maybeInjectWatermark();
if (prefetchingQueue.size() >= MAX_PREFETCHING_QUEUE_SIZE) {
- return PrefetchRoundResult.dormant();
+ return computeIdleRoundResult(walScanBudget);
}
if (walScanBudget.hasYielded()
|| remainingEntryBudget[0] <= 0
@@ -2591,7 +2591,10 @@ public class ConsensusPrefetchingQueue {
return PrefetchRoundResult.dormant();
}
if (hasReachedPrefetchLimit()) {
- return PrefetchRoundResult.dormant();
+ // Backpressure is temporary: writes may have stopped, and another queue
releasing its
+ // memory quota does not wake this queue. Keep a delayed retry without
scanning more WAL
+ // or immediately rescheduling while the delivery queue is still full.
+ return PrefetchRoundResult.rescheduleAfter(MEMORY_RETRY_SLEEP_MS);
}
if (hasImmediatePrefetchableWork(walScanBudget)) {
return PrefetchRoundResult.rescheduleNow();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWalBackpressureTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWalBackpressureTest.java
index 88474889ba7..590777dd4ff 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWalBackpressureTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWalBackpressureTest.java
@@ -45,6 +45,8 @@ import
org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileStatus;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils;
import org.apache.iotdb.db.subscription.event.SubscriptionEvent;
import org.apache.iotdb.db.subscription.resource.SubscriptionMemoryManager;
+import
org.apache.iotdb.db.subscription.task.execution.ConsensusSubscriptionPrefetchExecutor;
+import org.apache.iotdb.db.subscription.task.subtask.ConsensusPrefetchSubtask;
import org.apache.iotdb.rpc.subscription.config.TopicConstant;
import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress;
import org.apache.iotdb.rpc.subscription.payload.poll.WriterId;
@@ -63,9 +65,11 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Deque;
import java.util.List;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
@@ -80,6 +84,7 @@ import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
@@ -92,6 +97,196 @@ public class ConsensusPrefetchingQueueWalBackpressureTest {
@Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+ @Test
+ public void testPrefetchLimitRetriesWalAfterWritesStop() throws Exception {
+ assertPrefetchLimitRetriesAfterWritesStop(false, false);
+ }
+
+ @Test
+ public void testPrefetchLimitRetriesPendingAfterWritesStop() throws
Exception {
+ assertPrefetchLimitRetriesAfterWritesStop(true, false);
+ }
+
+ @Test
+ public void testPrefetchLimitResumesWhenAnotherQueueReleasesQuota() throws
Exception {
+ assertPrefetchLimitRetriesAfterWritesStop(false, true);
+ }
+
+ private void assertPrefetchLimitRetriesAfterWritesStop(
+ final boolean pendingSuffix, final boolean competingQueue) throws
Exception {
+ final String originalSystemDir =
IoTDBDescriptor.getInstance().getConfig().getSystemDir();
+ final CommonConfig config = CommonDescriptor.getInstance().getConfig();
+ final int originalBatchMaxWalEntries =
config.getSubscriptionConsensusBatchMaxWalEntries();
+ final int originalBatchMaxTabletCount =
config.getSubscriptionConsensusBatchMaxTabletCount();
+ final int originalBatchMaxDelay =
config.getSubscriptionConsensusBatchMaxDelayInMs();
+ final long originalWatermarkInterval =
config.getSubscriptionConsensusWatermarkIntervalMs();
+ final File systemDir =
temporaryFolder.newFolder("system-prefetch-limit-retry");
+ final File walDirectory =
temporaryFolder.newFolder("prefetch-limit-retry");
+ ConsensusPrefetchingQueue queue = null;
+ ConsensusPrefetchSubtask subtask = null;
+ SubscriptionMemoryManager.MemoryHandle otherQueue = null;
+ try {
+ config.setSubscriptionConsensusBatchMaxWalEntries(1);
+ config.setSubscriptionConsensusBatchMaxTabletCount(1);
+ config.setSubscriptionConsensusBatchMaxDelayInMs(0);
+ config.setSubscriptionConsensusWatermarkIntervalMs(0L);
+ writeSealedWal(walDirectory);
+
+ final WALNode walNode = mock(WALNode.class);
+ when(walNode.getLogDirectory()).thenReturn(walDirectory);
+ when(walNode.getCurrentSearchIndex()).thenReturn((long) REQUEST_COUNT);
+ when(walNode.getCurrentWALFileVersion()).thenReturn(1L);
+ when(walNode.getWALMetaDataSnapshot(anyLong())).thenReturn(new
WALMetaData());
+ final IoTConsensusServerImpl serverImpl =
mock(IoTConsensusServerImpl.class);
+ when(serverImpl.getConsensusReqReader()).thenReturn(walNode);
+ when(serverImpl.getWriterSafeFrontierTracker()).thenReturn(new
WriterSafeFrontierTracker());
+ final List<Long> conversions = new ArrayList<>();
+ final ConsensusLogToTabletConverter converter =
mock(ConsensusLogToTabletConverter.class);
+ when(converter.isTableModel()).thenReturn(true);
+ when(converter.getDatabaseName()).thenReturn("db");
+ when(converter.convert(any()))
+ .thenAnswer(
+ invocation -> {
+ conversions.add(((InsertNode)
invocation.getArgument(0)).getSearchIndex());
+ return Collections.singletonList(createTablet());
+ });
+ queue =
+ new ConsensusPrefetchingQueue(
+ "consumerGroup",
+ "topic",
+ TopicConstant.ORDER_MODE_LEADER_ONLY_VALUE,
+ new DataRegionId(1),
+ serverImpl,
+ new SubscriptionWalRetentionPolicy(
+ "topic",
+ SubscriptionWalRetentionPolicy.UNBOUNDED,
+ SubscriptionWalRetentionPolicy.UNBOUNDED),
+ converter,
+ newCommitManager(systemDir),
+ new RegionProgress(Collections.emptyMap()),
+ 1L,
+ 1L,
+ true);
+ final long oneTabletBytes =
PipeMemoryWeightUtil.calculateTabletSizeInBytes(createTablet());
+ final TrackingSubscriptionMemoryManager memoryManager =
+ new TrackingSubscriptionMemoryManager(oneTabletBytes *
(competingQueue ? 4L : 2L));
+ queue.setSubscriptionMemoryManager(memoryManager);
+ if (competingQueue) {
+ otherQueue = memoryManager.registerQueue();
+ otherQueue.setActive(true);
+ }
+ assertNull(queue.poll("consumer"));
+ assertTrue(pendingEntries(queue).offer(createRequest(1L)));
+ if (pendingSuffix) {
+ assertTrue(pendingEntries(queue).offer(createRequest(2L)));
+ assertTrue(pendingEntries(queue).offer(createRequest(3L)));
+ }
+ queue.drivePrefetchOnce();
+ assertEquals(1, queue.getPrefetchedEventCount());
+ assertEquals(oneTabletBytes, queue.getRetainedTabletBytes());
+ assertEquals(2L, queue.getCurrentReadSearchIndex());
+ assertEquals(0L, queue.getWalPathAcceptedEntries());
+
+ // Drive the real subtask with deterministic timers. No further writes
or offer callbacks
+ // occur, so every round below must be scheduled by the backpressured
queue itself.
+ final Deque<ConsensusPrefetchSubtask> ready = new ArrayDeque<>();
+ final Deque<Runnable> delayed = new ArrayDeque<>();
+ final ConsensusSubscriptionPrefetchExecutor executor =
+ mock(ConsensusSubscriptionPrefetchExecutor.class);
+ doAnswer(
+ invocation -> {
+ ready.addLast(invocation.getArgument(0));
+ return null;
+ })
+ .when(executor)
+ .enqueue(any());
+ doAnswer(
+ invocation -> {
+ final ConsensusPrefetchSubtask task =
invocation.getArgument(0);
+ final long delayMs = invocation.getArgument(1);
+ final long token = invocation.getArgument(2);
+ assertEquals(100L, delayMs);
+ delayed.addLast(() -> task.fireScheduledWakeup(token));
+ return null;
+ })
+ .when(executor)
+ .schedule(any(), anyLong(), anyLong());
+ subtask = new ConsensusPrefetchSubtask(queue);
+ subtask.bindExecutor(executor);
+ subtask.requestWakeupNow();
+ ready.removeFirst().runOneRound();
+ for (int retry = 0; retry < 2; retry++) {
+ assertTrue(ready.isEmpty());
+ assertEquals(1, delayed.size());
+ delayed.removeFirst().run();
+ ready.removeFirst().runOneRound();
+ assertEquals(Collections.singletonList(1L), conversions);
+ assertEquals(2L, queue.getCurrentReadSearchIndex());
+ assertMemoryBounded(queue, memoryManager);
+ }
+ if (competingQueue) {
+ // A group/topic disappearing changes the fair share without polling,
ACKing or offering
+ // anything to this queue. The outstanding timer must resume its
unread WAL on its own.
+ otherQueue.close();
+ otherQueue = null;
+ for (int round = 0; round < REQUEST_COUNT &&
queue.getPrefetchedEventCount() < 2; round++) {
+ runNextPrefetchRound(ready, delayed);
+ }
+ assertEquals(Arrays.asList(1L, 2L), conversions);
+ assertEquals(2, queue.getPrefetchedEventCount());
+ assertEquals(3L, queue.getCurrentReadSearchIndex());
+ }
+ for (long expected = 1L; expected <= REQUEST_COUNT; expected++) {
+ final SubscriptionEvent event = queue.poll("consumer");
+ assertNotNull(event);
+ assertEquals(expected,
event.getCommitContext().getWriterProgress().getLocalSeq());
+ assertTrue(queue.ack("consumer", event.getCommitContext()));
+ assertMemoryBounded(queue, memoryManager);
+ for (int round = 0;
+ expected < REQUEST_COUNT
+ && queue.getPrefetchedEventCount() == 0
+ && round < REQUEST_COUNT;
+ round++) {
+ runNextPrefetchRound(ready, delayed);
+ }
+ }
+ assertEquals(Arrays.asList(1L, 2L, 3L), conversions);
+ assertEquals(pendingSuffix ? 0L : 2L, queue.getWalPathAcceptedEntries());
+ assertEquals(pendingSuffix ? 3L : 1L,
queue.getPendingPathAcceptedEntries());
+ assertEquals(4L, queue.getCurrentReadSearchIndex());
+ assertEquals(0L, queue.getRetainedTabletBytes());
+ assertEquals(0L, memoryManager.getUsedMemorySizeInBytes());
+ assertEquals(0, queue.getPrefetchedEventCount());
+ assertEquals(0L, queue.getSubscriptionUncommittedEventCount());
+ assertEquals(PrefetchRoundResult.Type.DORMANT,
queue.drivePrefetchOnce().getType());
+ assertFalse(memoryManager.hasExceededBudget());
+ } finally {
+ if (subtask != null) {
+ subtask.close();
+ }
+ if (otherQueue != null) {
+ otherQueue.close();
+ }
+ if (queue != null) {
+ queue.close();
+ }
+
config.setSubscriptionConsensusBatchMaxWalEntries(originalBatchMaxWalEntries);
+
config.setSubscriptionConsensusBatchMaxTabletCount(originalBatchMaxTabletCount);
+ config.setSubscriptionConsensusBatchMaxDelayInMs(originalBatchMaxDelay);
+
config.setSubscriptionConsensusWatermarkIntervalMs(originalWatermarkInterval);
+
IoTDBDescriptor.getInstance().getConfig().setSystemDir(originalSystemDir);
+ }
+ }
+
+ private static void runNextPrefetchRound(
+ final Deque<ConsensusPrefetchSubtask> ready, final Deque<Runnable>
delayed) {
+ if (ready.isEmpty()) {
+ assertEquals(1, delayed.size());
+ delayed.removeFirst().run();
+ }
+ ready.removeFirst().runOneRound();
+ }
+
@Test
public void testYieldedGapScanKeepsIteratorAndDeliversEachRequestOnce()
throws Exception {
final String originalSystemDir =
IoTDBDescriptor.getInstance().getConfig().getSystemDir();