This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch fix/subscription-remote-wal-wakeup in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3a343faaff65424068f357819ecfe97c8a41ca9d Author: Caideyipi <[email protected]> AuthorDate: Mon Nov 9 17:21:08 2026 +0800 Throttle subscription prefetch empty-poll wakeups --- .../broker/consensus/ConsensusPrefetchingQueue.java | 21 ++++++++++++++++++++- .../ConsensusPrefetchingQueueSeekTest.java | 14 ++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java index 0f621637e3a..a93ac4172ab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java @@ -115,6 +115,8 @@ public class ConsensusPrefetchingQueue { private static final Logger LOGGER = LoggerFactory.getLogger(ConsensusPrefetchingQueue.class); + static final long EMPTY_POLL_PREFETCH_WAKEUP_INTERVAL_NANOS = TimeUnit.MILLISECONDS.toNanos(100L); + private static final ExecutorService CLOSE_CLEANUP_EXECUTOR = IoTDBThreadPoolFactory.newCachedThreadPoolWithDaemon( ThreadName.SUBSCRIPTION_CONSENSUS_PREFETCH_EXECUTOR_POOL.getName() + "-Close-Cleanup"); @@ -307,6 +309,8 @@ public class ConsensusPrefetchingQueue { private final Object prefetchBindingLock = new Object(); + private final AtomicLong lastEmptyPollPrefetchWakeupNanos = new AtomicLong(Long.MIN_VALUE); + private volatile ConsensusPrefetchSubtask prefetchSubtask; private volatile ConsensusSubscriptionPrefetchExecutor prefetchExecutor; @@ -964,7 +968,8 @@ public class ConsensusPrefetchingQueue { if (!bindingMissing) { // Replicated WAL entries do not update the local search index or enter the realtime queue. // Wake an idle worker on empty polls to discover entries that arrived after its last scan. - return !currentSubtask.isScheduledOrRunning(); + return !currentSubtask.isScheduledOrRunning() + && tryAcquireEmptyPollPrefetchWakeup(lastEmptyPollPrefetchWakeupNanos, System.nanoTime()); } return hasImmediatePrefetchableWork() @@ -975,6 +980,20 @@ public class ConsensusPrefetchingQueue { || computeWatermarkDelayMs() > 0L; } + static boolean tryAcquireEmptyPollPrefetchWakeup( + final AtomicLong lastWakeupNanos, final long currentTimeNanos) { + while (true) { + final long lastWakeupTimeNanos = lastWakeupNanos.get(); + if (lastWakeupTimeNanos != Long.MIN_VALUE + && currentTimeNanos - lastWakeupTimeNanos < EMPTY_POLL_PREFETCH_WAKEUP_INTERVAL_NANOS) { + return false; + } + if (lastWakeupNanos.compareAndSet(lastWakeupTimeNanos, currentTimeNanos)) { + return true; + } + } + } + // ======================== Poll ======================== public SubscriptionEvent poll(final String consumerId) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java index 6a9ba5e9cf5..d96d8536a77 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java @@ -67,6 +67,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -144,6 +145,19 @@ public class ConsensusPrefetchingQueueSeekTest { Arrays.asList(createRequest(1L, 1L, 7), createRequest(-1L, 10L, 8)), 2L, 0L, true); } + @Test + public void testEmptyPollWakeupRateLimit() { + final AtomicLong lastWakeupNanos = new AtomicLong(Long.MIN_VALUE); + assertTrue(ConsensusPrefetchingQueue.tryAcquireEmptyPollPrefetchWakeup(lastWakeupNanos, 0L)); + assertFalse( + ConsensusPrefetchingQueue.tryAcquireEmptyPollPrefetchWakeup( + lastWakeupNanos, + ConsensusPrefetchingQueue.EMPTY_POLL_PREFETCH_WAKEUP_INTERVAL_NANOS - 1L)); + assertTrue( + ConsensusPrefetchingQueue.tryAcquireEmptyPollPrefetchWakeup( + lastWakeupNanos, ConsensusPrefetchingQueue.EMPTY_POLL_PREFETCH_WAKEUP_INTERVAL_NANOS)); + } + @Test(timeout = 20_000L) public void testEmptyPollReplaysRemoteEntryAfterPrefetchBecomesDormant() throws Exception { final int originalBatchMaxDelay = config.getSubscriptionConsensusBatchMaxDelayInMs();
