This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 698448a36a2 Fix subscription replay ordering during writer handoff
(#19322)
698448a36a2 is described below
commit 698448a36a23579a8229c3de211805a02434b23d
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 19 15:44:14 2026 +0800
Fix subscription replay ordering during writer handoff (#19322)
---
.../consensus/ConsensusPrefetchingQueue.java | 101 ++++++++++++-----
...nsensusSubscriptionPrefetchingQueueMetrics.java | 10 ++
...onsensusPrefetchingQueueWriterOrderingTest.java | 124 ++++++++++++++++++++-
...susSubscriptionPrefetchingQueueMetricsTest.java | 19 ++++
.../iotdb/commons/service/metric/enums/Metric.java | 1 +
5 files changed, 226 insertions(+), 29 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
index 108e3afde88..8d77149264f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
@@ -270,7 +270,7 @@ public class ConsensusPrefetchingQueue {
private volatile int preferredWriterNodeId = -1;
- private volatile int previousPreferredWriterNodeId = -1;
+ private final Set<Integer> handoffWriterNodeIds = new LinkedHashSet<>();
// ======================== Routing Runtime Version ========================
@@ -328,6 +328,8 @@ public class ConsensusPrefetchingQueue {
private final Object runtimeActivationLock = new Object();
+ private final Object runtimeWriterStateLock = new Object();
+
/** Active runtime state waiting for an explicitly available ConfigNode
frontier. */
private volatile ConsensusRegionRuntimeState pendingActivationRuntimeState;
@@ -2067,6 +2069,7 @@ public class ConsensusPrefetchingQueue {
final int batchMaxDelayMs,
final ProgressWALIterator.ScanBudget walScanBudget)
throws IOException {
+ final Set<Integer> replayActiveWriterNodeIds = activeWriterNodeIds;
if (orderedReplayStartSearchIndex == Long.MIN_VALUE) {
orderedReplayStartSearchIndex = nextExpectedSearchIndex.get();
if (!lingerBatch.isEmpty()) {
@@ -2089,7 +2092,9 @@ public class ConsensusPrefetchingQueue {
}
final Map<Integer, SafeFrontier> safeFrontiers =
serverImpl.getWriterSafeFrontierTracker().snapshotSafeFrontiers();
- activeWriterNodeIds.forEach(this::getOrCreateWriterReplay);
+ for (final int writerNodeId : replayActiveWriterNodeIds) {
+ getOrCreateWriterReplay(writerNodeId).wasRuntimeActive = true;
+ }
safeFrontiers.keySet().forEach(this::getOrCreateWriterReplay);
recoveryWriterProgressByWriter.keySet().forEach(id ->
getOrCreateWriterReplay(id.getNodeId()));
realtimeEntriesByWriter.keySet().forEach(this::getOrCreateWriterReplay);
@@ -2124,6 +2129,9 @@ public class ConsensusPrefetchingQueue {
if (orderedWriterReplay.size() != writerCount && remainingEntryBudget[0]
> 0) {
continue;
}
+ if (replayActiveWriterNodeIds != activeWriterNodeIds) {
+ return PrefetchRoundResult.rescheduleNow();
+ }
final PriorityQueue<WriterFrontier> frontiers = new PriorityQueue<>();
for (final Map.Entry<Integer, WriterReplayState> writer :
orderedWriterReplay.entrySet()) {
@@ -2139,10 +2147,17 @@ public class ConsensusPrefetchingQueue {
writerNodeId,
new OrderingKey(head.getPhysicalTime(), writerNodeId,
head.getProgressLocalSeq()),
false));
- } else if (isWriterRuntimeActive(writerNodeId) || !state.caughtUp) {
+ } else {
final SafeFrontier safe = safeFrontiers.get(writerNodeId);
+ final boolean hasUnreadAppliedProgress =
+ Objects.nonNull(safe) && state.consumedLocalSeq <
safe.getLocalSeq();
+ if (!isWriterRuntimeActive(replayActiveWriterNodeIds, writerNodeId)
+ && state.caughtUp
+ && !(state.wasRuntimeActive && hasUnreadAppliedProgress)) {
+ continue;
+ }
final long safePt =
- Objects.nonNull(safe) && state.consumedLocalSeq >=
safe.getLocalSeq()
+ Objects.nonNull(safe) && safe.getPhysicalTime() > 0 &&
!hasUnreadAppliedProgress
? safe.getPhysicalTime()
: Long.MIN_VALUE;
frontiers.add(WriterFrontier.forBarrier(writerNodeId, safePt));
@@ -2422,6 +2437,7 @@ public class ConsensusPrefetchingQueue {
private IndexedConsensusRequest head;
private boolean fromPending;
private boolean caughtUp;
+ private boolean wasRuntimeActive;
private WriterReplayState(final ProgressWALIterator iterator, final long
consumedLocalSeq) {
this.iterator = iterator;
@@ -5693,6 +5709,10 @@ public class ConsensusPrefetchingQueue {
return isActive ? 1L : 0L;
}
+ public long getActiveWriterCount() {
+ return activeWriterNodeIds.size();
+ }
+
public long getInitializedStatus() {
return prefetchInitialized ? 1L : 0L;
}
@@ -5814,10 +5834,12 @@ public class ConsensusPrefetchingQueue {
}
public void setActiveWriterNodeIds(final Set<Integer> activeWriterNodeIds) {
- this.runtimeActiveWriterNodeIds =
- Collections.unmodifiableSet(
- new LinkedHashSet<>(Objects.requireNonNull(activeWriterNodeIds)));
- refreshEffectiveActiveWriterNodeIds();
+ synchronized (runtimeWriterStateLock) {
+ updateRuntimeActiveWritersUnderLock(
+ Collections.unmodifiableSet(
+ new
LinkedHashSet<>(Objects.requireNonNull(activeWriterNodeIds))));
+ refreshEffectiveActiveWriterNodeIds();
+ }
LOGGER.info(
DataNodePipeMessages
.PIPE_LOG_CONSENSUSPREFETCHINGQUEUE_RUNTIMEACTIVEWRITERNODEIDS_EFFECTIVEACTIVEWRITERNODEIDS_246519D2,
@@ -5830,6 +5852,11 @@ public class ConsensusPrefetchingQueue {
requestPrefetch();
}
+ private void updateRuntimeActiveWritersUnderLock(final Set<Integer>
writerNodeIds) {
+ runtimeActiveWriterNodeIds = writerNodeIds;
+ handoffWriterNodeIds.retainAll(writerNodeIds);
+ }
+
private void refreshEffectiveActiveWriterNodeIds() {
final LinkedHashSet<Integer> effectiveWriterNodeIds = new
LinkedHashSet<>();
switch (orderMode) {
@@ -5849,24 +5876,23 @@ public class ConsensusPrefetchingQueue {
if (preferredWriterNodeId >= 0) {
effectiveWriterNodeIds.add(preferredWriterNodeId);
}
- if (previousPreferredWriterNodeId >= 0
- && previousPreferredWriterNodeId != preferredWriterNodeId
- &&
runtimeActiveWriterNodeIds.contains(previousPreferredWriterNodeId)) {
- effectiveWriterNodeIds.add(previousPreferredWriterNodeId);
+ for (final int writerNodeId : handoffWriterNodeIds) {
+ if (runtimeActiveWriterNodeIds.contains(writerNodeId)) {
+ effectiveWriterNodeIds.add(writerNodeId);
+ }
}
break;
}
- this.activeWriterNodeIds =
Collections.unmodifiableSet(effectiveWriterNodeIds);
+ if (!activeWriterNodeIds.equals(effectiveWriterNodeIds)) {
+ this.activeWriterNodeIds =
Collections.unmodifiableSet(effectiveWriterNodeIds);
+ }
}
public void setPreferredWriterNodeId(final int preferredWriterNodeId) {
- if (this.preferredWriterNodeId != preferredWriterNodeId) {
- previousPreferredWriterNodeId = this.preferredWriterNodeId;
- } else {
- previousPreferredWriterNodeId = -1;
+ synchronized (runtimeWriterStateLock) {
+ updatePreferredWriterUnderLock(preferredWriterNodeId);
+ refreshEffectiveActiveWriterNodeIds();
}
- this.preferredWriterNodeId = preferredWriterNodeId;
- refreshEffectiveActiveWriterNodeIds();
LOGGER.info(
DataNodePipeMessages
.PIPE_LOG_CONSENSUSPREFETCHINGQUEUE_PREFERREDWRITERNODEID_SET_TO_EFFECTIVEACTIVEWRITERNODEIDS_B08E8180,
@@ -5878,17 +5904,29 @@ public class ConsensusPrefetchingQueue {
requestPrefetch();
}
+ private void updatePreferredWriterUnderLock(final int writerNodeId) {
+ if (preferredWriterNodeId != writerNodeId) {
+ if (preferredWriterNodeId >= 0) {
+ handoffWriterNodeIds.add(preferredWriterNodeId);
+ }
+ handoffWriterNodeIds.remove(writerNodeId);
+ preferredWriterNodeId = writerNodeId;
+ }
+ }
+
public Set<Integer> getActiveWriterNodeIds() {
return activeWriterNodeIds;
}
public void setOrderMode(final String orderMode) {
final String normalizedOrderMode =
TopicConfig.normalizeOrderMode(orderMode);
- if (Objects.equals(this.orderMode, normalizedOrderMode)) {
- return;
+ synchronized (runtimeWriterStateLock) {
+ if (Objects.equals(this.orderMode, normalizedOrderMode)) {
+ return;
+ }
+ this.orderMode = normalizedOrderMode;
+ refreshEffectiveActiveWriterNodeIds();
}
- this.orderMode = normalizedOrderMode;
- refreshEffectiveActiveWriterNodeIds();
LOGGER.info(
DataNodePipeMessages
.PIPE_LOG_CONSENSUSPREFETCHINGQUEUE_ORDERMODE_SET_TO_EFFECTIVEACTIVEWRITERNODEIDS_CDD3C86E,
@@ -5906,7 +5944,11 @@ public class ConsensusPrefetchingQueue {
}
private boolean isWriterRuntimeActive(final int writerNodeId) {
- final Set<Integer> writerNodeIds = activeWriterNodeIds;
+ return isWriterRuntimeActive(activeWriterNodeIds, writerNodeId);
+ }
+
+ private static boolean isWriterRuntimeActive(
+ final Set<Integer> writerNodeIds, final int writerNodeId) {
return writerNodeIds.isEmpty() || writerNodeIds.contains(writerNodeId);
}
@@ -5926,8 +5968,11 @@ public class ConsensusPrefetchingQueue {
DataNodePipeMessages.PIPE_LOG_CONSENSUSPREFETCHINGQUEUE_APPLIED_RUNTIMEVERSION_36E05B80,
this,
runtimeState.getRuntimeVersion());
- setPreferredWriterNodeId(runtimeState.getPreferredWriterNodeId());
- setActiveWriterNodeIds(runtimeState.getActiveWriterNodeIds());
+ synchronized (runtimeWriterStateLock) {
+
updatePreferredWriterUnderLock(runtimeState.getPreferredWriterNodeId());
+
updateRuntimeActiveWritersUnderLock(runtimeState.getActiveWriterNodeIds());
+ refreshEffectiveActiveWriterNodeIds();
+ }
// "active" decides whether this replica should serve subscription
traffic on the current
// node.
// In multi-writer mode, activeWriterNodeIds may intentionally include
follower replicas for
@@ -5937,7 +5982,9 @@ public class ConsensusPrefetchingQueue {
pendingActivationRuntimeState = null;
}
if (requiresAuthoritativeActivation) {
- tryActivatePendingRuntimeState();
+ if (!tryActivatePendingRuntimeState()) {
+ requestPrefetch();
+ }
} else {
setActive(runtimeState.isActive() &&
Objects.isNull(pendingActivationRuntimeState));
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/metric/ConsensusSubscriptionPrefetchingQueueMetrics.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/metric/ConsensusSubscriptionPrefetchingQueueMetrics.java
index f0c3cfed01d..9decf5b489c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/metric/ConsensusSubscriptionPrefetchingQueueMetrics.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/metric/ConsensusSubscriptionPrefetchingQueueMetrics.java
@@ -146,6 +146,12 @@ public class ConsensusSubscriptionPrefetchingQueueMetrics
implements IMetricSet
queue,
ConsensusPrefetchingQueue::getActiveStatus,
key.getTags());
+ metricService.createAutoGauge(
+ Metric.SUBSCRIPTION_CONSENSUS_ACTIVE_WRITER_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ queue,
+ ConsensusPrefetchingQueue::getActiveWriterCount,
+ key.getTags());
metricService.createAutoGauge(
Metric.SUBSCRIPTION_CONSENSUS_INITIALIZED.toString(),
MetricLevel.IMPORTANT,
@@ -373,6 +379,10 @@ public class ConsensusSubscriptionPrefetchingQueueMetrics
implements IMetricSet
MetricType.AUTO_GAUGE,
Metric.SUBSCRIPTION_CONSENSUS_WATERMARK.toString(), key.getTags());
metricService.remove(
MetricType.AUTO_GAUGE,
Metric.SUBSCRIPTION_CONSENSUS_ACTIVE.toString(), key.getTags());
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.SUBSCRIPTION_CONSENSUS_ACTIVE_WRITER_COUNT.toString(),
+ key.getTags());
metricService.remove(
MetricType.AUTO_GAUGE,
Metric.SUBSCRIPTION_CONSENSUS_INITIALIZED.toString(), key.getTags());
metricService.remove(
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWriterOrderingTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWriterOrderingTest.java
index 972a3a5fdc0..983c254c207 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWriterOrderingTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueWriterOrderingTest.java
@@ -236,6 +236,113 @@ public class ConsensusPrefetchingQueueWriterOrderingTest {
assertEvent(7, 2000);
}
+ @Test
+ public void testLeaderOnlyReplayDropsPreviousWriterAfterRuntimeRetiresIt()
throws Exception {
+ newQueue(
+ mock(ConsensusReqReader.class), 1024 * 1024,
TopicConstant.ORDER_MODE_LEADER_ONLY_VALUE);
+ queue.applyRuntimeState(new ConsensusRegionRuntimeState(2, 5, true,
Set.of(1, 5)));
+ queue.applyRuntimeState(new ConsensusRegionRuntimeState(3, 1, true,
Set.of(1, 5)));
+ tracker.observePendingWriterSafeTimeBarrier(3000, 5, 0);
+ assertEquals(Set.of(1, 5), queue.getActiveWriterNodeIds());
+ assertEquals(2L, queue.getActiveWriterCount());
+ assertTrue(pending().offer(request(1, 1, 2000)));
+
+ queue.drivePrefetchOnce();
+
+ assertEvent(1, 2000);
+ assertTrue(pending().offer(request(2, 1, 4000)));
+ queue.drivePrefetchOnce();
+ assertNull(queue.poll("consumer"));
+ assertEquals(Set.of(1, 5), queue.getActiveWriterNodeIds());
+
+ queue.applyRuntimeState(ConsensusRegionRuntimeState.leaderOnly(4, 1,
true));
+ queue.drivePrefetchOnce();
+
+ assertEvent(1, 4000);
+ assertEquals(Set.of(1), queue.getActiveWriterNodeIds());
+ assertEquals(1L, queue.getActiveWriterCount());
+ }
+
+ @Test
+ public void
testLeaderOnlyReplayKeepsPreviousWriterUntilSafeFrontierIsConsumed()
+ throws Exception {
+ final File directory =
temporaryFolder.newFolder("leader-only-delayed-wal");
+ newQueue(walNode(directory, 1), 1024 * 1024,
TopicConstant.ORDER_MODE_LEADER_ONLY_VALUE);
+ queue.applyRuntimeState(new ConsensusRegionRuntimeState(2, 5, true,
Set.of(1, 5)));
+ queue.applyRuntimeState(new ConsensusRegionRuntimeState(3, 1, true,
Set.of(1, 5)));
+ assertTrue(pending().offer(request(1, 1, 2000)));
+ queue.drivePrefetchOnce();
+ assertNull(queue.poll("consumer"));
+
+ tracker.recordAppliedProgress(1000, 5, 1);
+ tracker.observePendingWriterSafeTimeBarrier(3000, 5, 1);
+
+ assertEquals(PrefetchRoundResult.Type.RESCHEDULE_LATER,
queue.drivePrefetchOnce().getType());
+ assertNull(queue.poll("consumer"));
+ assertEquals(Set.of(1, 5), queue.getActiveWriterNodeIds());
+
+ queue.applyRuntimeState(ConsensusRegionRuntimeState.leaderOnly(4, 1,
true));
+ queue.drivePrefetchOnce();
+ assertNull(queue.poll("consumer"));
+
+ final File data =
+ new File(directory, WALFileUtils.getLogFileName(0, 0,
WALFileStatus.CONTAINS_SEARCH_INDEX));
+ try (WALWriter writer = new WALWriter(data, WALFileVersion.V3)) {
+ writeEntry(writer, -1, 5, 1000, 1);
+ }
+ queue.drivePrefetchOnce();
+
+ assertEvent(5, 1000);
+ assertEvent(1, 2000);
+ assertNull(queue.poll("consumer"));
+ assertEquals(2, conversions.get());
+ assertEquals(Set.of(1), queue.getActiveWriterNodeIds());
+ }
+
+ @Test
+ public void testLeaderOnlyReplayKeepsHandoffWritersAcrossRuntimeRefresh()
throws Exception {
+ newQueue(
+ mock(ConsensusReqReader.class), 1024 * 1024,
TopicConstant.ORDER_MODE_LEADER_ONLY_VALUE);
+ final Set<Integer> writers = Set.of(1, 5, 7);
+ queue.applyRuntimeState(new ConsensusRegionRuntimeState(2, 5, true,
writers));
+ queue.applyRuntimeState(new ConsensusRegionRuntimeState(3, 1, true,
writers));
+ queue.applyRuntimeState(new ConsensusRegionRuntimeState(4, 7, true,
writers));
+ queue.applyRuntimeState(new ConsensusRegionRuntimeState(5, 7, true,
writers));
+ tracker.observePendingWriterSafeTimeBarrier(3000, 1, 0);
+ tracker.observePendingWriterSafeTimeBarrier(1000, 5, 0);
+ assertTrue(pending().offer(request(1, 7, 2000)));
+
+ queue.drivePrefetchOnce();
+
+ assertNull(queue.poll("consumer"));
+ assertEquals(writers, queue.getActiveWriterNodeIds());
+
+ tracker.recordAppliedProgress(1500, 5, 1);
+ tracker.observePendingWriterSafeTimeBarrier(3000, 5, 1);
+ assertTrue(pending().offer(request(-1, 5, 1500, 1)));
+ queue.drivePrefetchOnce();
+
+ assertEvent(5, 1500);
+ assertEvent(7, 2000);
+ assertNull(queue.poll("consumer"));
+ }
+
+ @Test
+ public void
testLeaderOnlyReplayKeepsPreviousWriterWhenSafeTimeIsUnavailable() throws
Exception {
+ newQueue(
+ mock(ConsensusReqReader.class), 1024 * 1024,
TopicConstant.ORDER_MODE_LEADER_ONLY_VALUE);
+ queue.setActiveWriterNodeIds(Set.of(5));
+ queue.setPreferredWriterNodeId(5);
+ queue.setPreferredWriterNodeId(1);
+ tracker.recordAppliedProgress(0, 5, 0);
+ assertTrue(pending().offer(request(1, 1, 0)));
+
+ queue.drivePrefetchOnce();
+
+ assertNull(queue.poll("consumer"));
+ assertEquals(Set.of(1, 5), queue.getActiveWriterNodeIds());
+ }
+
@Test
public void testDefaultPendingBatchBoundaryKeepsWriterOrdering() throws
Exception {
config.setSubscriptionConsensusBatchMaxDelayInMs(50);
@@ -474,6 +581,11 @@ public class ConsensusPrefetchingQueueWriterOrderingTest {
}
private void newQueue(final ConsensusReqReader reader, final long
memoryBytes) {
+ newQueue(reader, memoryBytes, TopicConstant.ORDER_MODE_MULTI_WRITER_VALUE);
+ }
+
+ private void newQueue(
+ final ConsensusReqReader reader, final long memoryBytes, final String
orderMode) {
final IoTConsensusServerImpl server = mock(IoTConsensusServerImpl.class);
when(server.getConsensusReqReader()).thenReturn(reader);
when(server.getWriterSafeFrontierTracker()).thenReturn(tracker);
@@ -489,7 +601,7 @@ public class ConsensusPrefetchingQueueWriterOrderingTest {
new ConsensusPrefetchingQueue(
"consumerGroup",
"topic",
- TopicConstant.ORDER_MODE_MULTI_WRITER_VALUE,
+ orderMode,
new DataRegionId(1),
server,
new SubscriptionWalRetentionPolicy(
@@ -543,9 +655,17 @@ public class ConsensusPrefetchingQueueWriterOrderingTest {
private static IndexedConsensusRequest request(
final long searchIndex, final int writerNodeId, final long physicalTime)
{
+ return request(searchIndex, writerNodeId, physicalTime, writerNodeId == 8
? 1 : searchIndex);
+ }
+
+ private static IndexedConsensusRequest request(
+ final long searchIndex,
+ final int writerNodeId,
+ final long physicalTime,
+ final long localSeq) {
return new IndexedConsensusRequest(
searchIndex,
- writerNodeId == 8 ? 1 : searchIndex,
+ localSeq,
Collections.singletonList(StatementTestUtils.genInsertRowNode(1)))
.setNodeId(writerNodeId)
.setPhysicalTime(physicalTime);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/metric/ConsensusSubscriptionPrefetchingQueueMetricsTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/metric/ConsensusSubscriptionPrefetchingQueueMetricsTest.java
index 8038d9cb0ad..f50143f17d9 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/metric/ConsensusSubscriptionPrefetchingQueueMetricsTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/metric/ConsensusSubscriptionPrefetchingQueueMetricsTest.java
@@ -199,6 +199,17 @@ public class
ConsensusSubscriptionPrefetchingQueueMetricsTest {
eq(Tag.REGION.toString()),
eq(secondRegionId.toString()));
+ verify(metricService)
+ .createAutoGauge(
+ eq(Metric.SUBSCRIPTION_CONSENSUS_ACTIVE_WRITER_COUNT.toString()),
+ eq(MetricLevel.IMPORTANT),
+ eq(firstQueue),
+ any(),
+ eq(Tag.NAME.toString()),
+ eq(queueId),
+ eq(Tag.REGION.toString()),
+ eq(firstRegionId.toString()));
+
metrics.mark(queueId, firstRegionId.toString(), 11L);
metrics.mark(queueId, secondRegionId.toString(), 22L);
verify(firstRate).mark(11L);
@@ -242,6 +253,14 @@ public class
ConsensusSubscriptionPrefetchingQueueMetricsTest {
queueId,
Tag.REGION.toString(),
firstRegionId.toString());
+ verify(metricService)
+ .remove(
+ MetricType.AUTO_GAUGE,
+ Metric.SUBSCRIPTION_CONSENSUS_ACTIVE_WRITER_COUNT.toString(),
+ Tag.NAME.toString(),
+ queueId,
+ Tag.REGION.toString(),
+ firstRegionId.toString());
verify(metricService)
.remove(
MetricType.AUTO_GAUGE,
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
index a4b853ddc9d..78123946be2 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -229,6 +229,7 @@ public enum Metric {
SUBSCRIPTION_CONSENSUS_WATERMARK("subscription_consensus_watermark"),
SUBSCRIPTION_CONSENSUS_SEEK_GENERATION("subscription_consensus_seek_generation"),
SUBSCRIPTION_CONSENSUS_ACTIVE("subscription_consensus_active"),
+
SUBSCRIPTION_CONSENSUS_ACTIVE_WRITER_COUNT("subscription_consensus_active_writer_count"),
SUBSCRIPTION_CONSENSUS_INITIALIZED("subscription_consensus_initialized"),
SUBSCRIPTION_CONSENSUS_REMAINING_EVENT_COUNT("subscription_consensus_remaining_event_count"),
SUBSCRIPTION_CONSENSUS_RAW_WAL_GAP("subscription_consensus_raw_wal_gap"),