This is an automated email from the ASF dual-hosted git repository.

Caideyipi pushed a commit to branch codex/subscription-prefetch-backpressure
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit eb68cd1ba5802d018238dd9c8fe10fb84c2cb2b6
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 5 12:52:22 2026 +0800

    Fix subscription prefetch retries under delivery backpressure
---
 .../consensus/ConsensusPrefetchingQueue.java       |   7 +-
 ...nsensusPrefetchingQueueWalBackpressureTest.java | 195 +++++++++++++++++++++
 2 files changed, 200 insertions(+), 2 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
index 5bcfd58d52d..765202b74f2 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
@@ -2177,7 +2177,7 @@ public class ConsensusPrefetchingQueue {
     }
     maybeInjectWatermark();
     if (prefetchingQueue.size() >= MAX_PREFETCHING_QUEUE_SIZE) {
-      return PrefetchRoundResult.dormant();
+      return computeIdleRoundResult(walScanBudget);
     }
     if (walScanBudget.hasYielded()
         || remainingEntryBudget[0] <= 0
@@ -2591,7 +2591,10 @@ public class ConsensusPrefetchingQueue {
       return PrefetchRoundResult.dormant();
     }
     if (hasReachedPrefetchLimit()) {
-      return PrefetchRoundResult.dormant();
+      // Backpressure is temporary: writes may have stopped, and another queue 
releasing its
+      // memory quota does not wake this queue. Keep a delayed retry without 
scanning more WAL
+      // or immediately rescheduling while the delivery queue is still full.
+      return PrefetchRoundResult.rescheduleAfter(MEMORY_RETRY_SLEEP_MS);
     }
     if (hasImmediatePrefetchableWork(walScanBudget)) {
       return PrefetchRoundResult.rescheduleNow();
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWalBackpressureTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWalBackpressureTest.java
index 88474889ba7..590777dd4ff 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWalBackpressureTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWalBackpressureTest.java
@@ -45,6 +45,8 @@ import 
org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileStatus;
 import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils;
 import org.apache.iotdb.db.subscription.event.SubscriptionEvent;
 import org.apache.iotdb.db.subscription.resource.SubscriptionMemoryManager;
+import 
org.apache.iotdb.db.subscription.task.execution.ConsensusSubscriptionPrefetchExecutor;
+import org.apache.iotdb.db.subscription.task.subtask.ConsensusPrefetchSubtask;
 import org.apache.iotdb.rpc.subscription.config.TopicConstant;
 import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress;
 import org.apache.iotdb.rpc.subscription.payload.poll.WriterId;
@@ -63,9 +65,11 @@ import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Deque;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
@@ -80,6 +84,7 @@ import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
@@ -92,6 +97,196 @@ public class ConsensusPrefetchingQueueWalBackpressureTest {
 
   @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
 
+  @Test
+  public void testPrefetchLimitRetriesWalAfterWritesStop() throws Exception {
+    assertPrefetchLimitRetriesAfterWritesStop(false, false);
+  }
+
+  @Test
+  public void testPrefetchLimitRetriesPendingAfterWritesStop() throws 
Exception {
+    assertPrefetchLimitRetriesAfterWritesStop(true, false);
+  }
+
+  @Test
+  public void testPrefetchLimitResumesWhenAnotherQueueReleasesQuota() throws 
Exception {
+    assertPrefetchLimitRetriesAfterWritesStop(false, true);
+  }
+
+  private void assertPrefetchLimitRetriesAfterWritesStop(
+      final boolean pendingSuffix, final boolean competingQueue) throws 
Exception {
+    final String originalSystemDir = 
IoTDBDescriptor.getInstance().getConfig().getSystemDir();
+    final CommonConfig config = CommonDescriptor.getInstance().getConfig();
+    final int originalBatchMaxWalEntries = 
config.getSubscriptionConsensusBatchMaxWalEntries();
+    final int originalBatchMaxTabletCount = 
config.getSubscriptionConsensusBatchMaxTabletCount();
+    final int originalBatchMaxDelay = 
config.getSubscriptionConsensusBatchMaxDelayInMs();
+    final long originalWatermarkInterval = 
config.getSubscriptionConsensusWatermarkIntervalMs();
+    final File systemDir = 
temporaryFolder.newFolder("system-prefetch-limit-retry");
+    final File walDirectory = 
temporaryFolder.newFolder("prefetch-limit-retry");
+    ConsensusPrefetchingQueue queue = null;
+    ConsensusPrefetchSubtask subtask = null;
+    SubscriptionMemoryManager.MemoryHandle otherQueue = null;
+    try {
+      config.setSubscriptionConsensusBatchMaxWalEntries(1);
+      config.setSubscriptionConsensusBatchMaxTabletCount(1);
+      config.setSubscriptionConsensusBatchMaxDelayInMs(0);
+      config.setSubscriptionConsensusWatermarkIntervalMs(0L);
+      writeSealedWal(walDirectory);
+
+      final WALNode walNode = mock(WALNode.class);
+      when(walNode.getLogDirectory()).thenReturn(walDirectory);
+      when(walNode.getCurrentSearchIndex()).thenReturn((long) REQUEST_COUNT);
+      when(walNode.getCurrentWALFileVersion()).thenReturn(1L);
+      when(walNode.getWALMetaDataSnapshot(anyLong())).thenReturn(new 
WALMetaData());
+      final IoTConsensusServerImpl serverImpl = 
mock(IoTConsensusServerImpl.class);
+      when(serverImpl.getConsensusReqReader()).thenReturn(walNode);
+      when(serverImpl.getWriterSafeFrontierTracker()).thenReturn(new 
WriterSafeFrontierTracker());
+      final List<Long> conversions = new ArrayList<>();
+      final ConsensusLogToTabletConverter converter = 
mock(ConsensusLogToTabletConverter.class);
+      when(converter.isTableModel()).thenReturn(true);
+      when(converter.getDatabaseName()).thenReturn("db");
+      when(converter.convert(any()))
+          .thenAnswer(
+              invocation -> {
+                conversions.add(((InsertNode) 
invocation.getArgument(0)).getSearchIndex());
+                return Collections.singletonList(createTablet());
+              });
+      queue =
+          new ConsensusPrefetchingQueue(
+              "consumerGroup",
+              "topic",
+              TopicConstant.ORDER_MODE_LEADER_ONLY_VALUE,
+              new DataRegionId(1),
+              serverImpl,
+              new SubscriptionWalRetentionPolicy(
+                  "topic",
+                  SubscriptionWalRetentionPolicy.UNBOUNDED,
+                  SubscriptionWalRetentionPolicy.UNBOUNDED),
+              converter,
+              newCommitManager(systemDir),
+              new RegionProgress(Collections.emptyMap()),
+              1L,
+              1L,
+              true);
+      final long oneTabletBytes = 
PipeMemoryWeightUtil.calculateTabletSizeInBytes(createTablet());
+      final TrackingSubscriptionMemoryManager memoryManager =
+          new TrackingSubscriptionMemoryManager(oneTabletBytes * 
(competingQueue ? 4L : 2L));
+      queue.setSubscriptionMemoryManager(memoryManager);
+      if (competingQueue) {
+        otherQueue = memoryManager.registerQueue();
+        otherQueue.setActive(true);
+      }
+      assertNull(queue.poll("consumer"));
+      assertTrue(pendingEntries(queue).offer(createRequest(1L)));
+      if (pendingSuffix) {
+        assertTrue(pendingEntries(queue).offer(createRequest(2L)));
+        assertTrue(pendingEntries(queue).offer(createRequest(3L)));
+      }
+      queue.drivePrefetchOnce();
+      assertEquals(1, queue.getPrefetchedEventCount());
+      assertEquals(oneTabletBytes, queue.getRetainedTabletBytes());
+      assertEquals(2L, queue.getCurrentReadSearchIndex());
+      assertEquals(0L, queue.getWalPathAcceptedEntries());
+
+      // Drive the real subtask with deterministic timers. No further writes 
or offer callbacks
+      // occur, so every round below must be scheduled by the backpressured 
queue itself.
+      final Deque<ConsensusPrefetchSubtask> ready = new ArrayDeque<>();
+      final Deque<Runnable> delayed = new ArrayDeque<>();
+      final ConsensusSubscriptionPrefetchExecutor executor =
+          mock(ConsensusSubscriptionPrefetchExecutor.class);
+      doAnswer(
+              invocation -> {
+                ready.addLast(invocation.getArgument(0));
+                return null;
+              })
+          .when(executor)
+          .enqueue(any());
+      doAnswer(
+              invocation -> {
+                final ConsensusPrefetchSubtask task = 
invocation.getArgument(0);
+                final long delayMs = invocation.getArgument(1);
+                final long token = invocation.getArgument(2);
+                assertEquals(100L, delayMs);
+                delayed.addLast(() -> task.fireScheduledWakeup(token));
+                return null;
+              })
+          .when(executor)
+          .schedule(any(), anyLong(), anyLong());
+      subtask = new ConsensusPrefetchSubtask(queue);
+      subtask.bindExecutor(executor);
+      subtask.requestWakeupNow();
+      ready.removeFirst().runOneRound();
+      for (int retry = 0; retry < 2; retry++) {
+        assertTrue(ready.isEmpty());
+        assertEquals(1, delayed.size());
+        delayed.removeFirst().run();
+        ready.removeFirst().runOneRound();
+        assertEquals(Collections.singletonList(1L), conversions);
+        assertEquals(2L, queue.getCurrentReadSearchIndex());
+        assertMemoryBounded(queue, memoryManager);
+      }
+      if (competingQueue) {
+        // A group/topic disappearing changes the fair share without polling, 
ACKing or offering
+        // anything to this queue. The outstanding timer must resume its 
unread WAL on its own.
+        otherQueue.close();
+        otherQueue = null;
+        for (int round = 0; round < REQUEST_COUNT && 
queue.getPrefetchedEventCount() < 2; round++) {
+          runNextPrefetchRound(ready, delayed);
+        }
+        assertEquals(Arrays.asList(1L, 2L), conversions);
+        assertEquals(2, queue.getPrefetchedEventCount());
+        assertEquals(3L, queue.getCurrentReadSearchIndex());
+      }
+      for (long expected = 1L; expected <= REQUEST_COUNT; expected++) {
+        final SubscriptionEvent event = queue.poll("consumer");
+        assertNotNull(event);
+        assertEquals(expected, 
event.getCommitContext().getWriterProgress().getLocalSeq());
+        assertTrue(queue.ack("consumer", event.getCommitContext()));
+        assertMemoryBounded(queue, memoryManager);
+        for (int round = 0;
+            expected < REQUEST_COUNT
+                && queue.getPrefetchedEventCount() == 0
+                && round < REQUEST_COUNT;
+            round++) {
+          runNextPrefetchRound(ready, delayed);
+        }
+      }
+      assertEquals(Arrays.asList(1L, 2L, 3L), conversions);
+      assertEquals(pendingSuffix ? 0L : 2L, queue.getWalPathAcceptedEntries());
+      assertEquals(pendingSuffix ? 3L : 1L, 
queue.getPendingPathAcceptedEntries());
+      assertEquals(4L, queue.getCurrentReadSearchIndex());
+      assertEquals(0L, queue.getRetainedTabletBytes());
+      assertEquals(0L, memoryManager.getUsedMemorySizeInBytes());
+      assertEquals(0, queue.getPrefetchedEventCount());
+      assertEquals(0L, queue.getSubscriptionUncommittedEventCount());
+      assertEquals(PrefetchRoundResult.Type.DORMANT, 
queue.drivePrefetchOnce().getType());
+      assertFalse(memoryManager.hasExceededBudget());
+    } finally {
+      if (subtask != null) {
+        subtask.close();
+      }
+      if (otherQueue != null) {
+        otherQueue.close();
+      }
+      if (queue != null) {
+        queue.close();
+      }
+      
config.setSubscriptionConsensusBatchMaxWalEntries(originalBatchMaxWalEntries);
+      
config.setSubscriptionConsensusBatchMaxTabletCount(originalBatchMaxTabletCount);
+      config.setSubscriptionConsensusBatchMaxDelayInMs(originalBatchMaxDelay);
+      
config.setSubscriptionConsensusWatermarkIntervalMs(originalWatermarkInterval);
+      
IoTDBDescriptor.getInstance().getConfig().setSystemDir(originalSystemDir);
+    }
+  }
+
+  private static void runNextPrefetchRound(
+      final Deque<ConsensusPrefetchSubtask> ready, final Deque<Runnable> 
delayed) {
+    if (ready.isEmpty()) {
+      assertEquals(1, delayed.size());
+      delayed.removeFirst().run();
+    }
+    ready.removeFirst().runOneRound();
+  }
+
   @Test
   public void testYieldedGapScanKeepsIteratorAndDeliversEachRequestOnce() 
throws Exception {
     final String originalSystemDir = 
IoTDBDescriptor.getInstance().getConfig().getSystemDir();

Reply via email to