This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch codex/queues-bound-readiness in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7c3eade1aa11d474aa50fd5a1c3dc827e5609c97 Author: Caideyipi <[email protected]> AuthorDate: Tue Nov 17 15:34:48 2026 +0800 Expose consensus subscription queue readiness --- .../iotdb/confignode/manager/ConfigManager.java | 28 +++++- .../procedure/env/ConfigNodeProcedureEnv.java | 9 +- .../impl/DataNodeInternalRPCServiceImpl.java | 20 ++++- .../sys/subscription/ShowSubscriptionsTask.java | 6 ++ .../ConsensusSubscriptionSetupHandler.java | 99 ++++++++++++++++++++++ .../subscription/ShowSubscriptionsTaskTest.java | 43 +++++----- .../ConsensusSubscriptionSetupHandlerTest.java | 23 +++++ .../schema/column/ColumnHeaderConstant.java | 4 +- .../src/main/thrift/confignode.thrift | 1 + .../src/main/thrift/datanode.thrift | 9 ++ 10 files changed, 217 insertions(+), 25 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index cb9dab67bc6..daaa62cab62 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -274,6 +274,7 @@ import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType; import org.apache.iotdb.db.schemaengine.template.alter.TemplateAlterOperationUtil; +import org.apache.iotdb.mpp.rpc.thrift.TConsensusSubscriptionQueueReadiness; import org.apache.iotdb.mpp.rpc.thrift.TPullCommitProgressResp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -310,6 +311,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -2828,11 +2830,28 @@ public class ConfigManager implements IManager { info.getTopicName() + "\u0000" + info.getConsumerGroupId())); } final List<TSubscriptionProgressInfo> progressInfoList = new ArrayList<>(); + final Map<String, Set<Integer>> queuesBoundDataNodeIds = new HashMap<>(); final Map<Integer, TPullCommitProgressResp> dataNodeResponses = - getProcedureManager().getEnv().pullCommitProgressFromDataNodesBestEffort(); + getProcedureManager().getEnv().pullCommitProgressFromDataNodesBestEffort(true); for (final Map.Entry<Integer, TPullCommitProgressResp> dataNodeEntry : dataNodeResponses.entrySet()) { final TPullCommitProgressResp dataNodeResponse = dataNodeEntry.getValue(); + if (dataNodeResponse.isSetStatus() + && dataNodeResponse.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + && dataNodeResponse.isSetConsensusQueueReadiness()) { + for (final TConsensusSubscriptionQueueReadiness readiness : + dataNodeResponse.getConsensusQueueReadiness()) { + if (readiness.isTableModel() != (req.isSetIsTableModel() && req.isIsTableModel())) { + continue; + } + final String key = readiness.getTopicName() + "\u0000" + readiness.getConsumerGroupId(); + if (visibleSubscriptions.contains(key)) { + queuesBoundDataNodeIds + .computeIfAbsent(key, ignored -> new HashSet<>()) + .add(dataNodeEntry.getKey()); + } + } + } if (!dataNodeResponse.isSetSubscriptionProgress()) { continue; } @@ -2917,6 +2936,13 @@ public class ConfigManager implements IManager { progressInfo.getConsumerGroupId(), tableModel, registeredDataNodeIds)); + final Set<Integer> queuesBoundIds = + new TreeSet<>( + queuesBoundDataNodeIds.getOrDefault( + progressInfo.getTopicName() + "\u0000" + progressInfo.getConsumerGroupId(), + Collections.emptySet())); + queuesBoundIds.retainAll(registeredDataNodeIds); + progressInfo.setQueuesBoundOnDns(new ArrayList<>(queuesBoundIds)); } progressInfoList.sort( Comparator.comparing(TSubscriptionProgressInfo::getTopicName) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java index 8fd27ab0fc0..eb01e326a39 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java @@ -921,9 +921,16 @@ public class ConfigNodeProcedureEnv { } public Map<Integer, TPullCommitProgressResp> pullCommitProgressFromDataNodesBestEffort() { + return pullCommitProgressFromDataNodesBestEffort(false); + } + + public Map<Integer, TPullCommitProgressResp> pullCommitProgressFromDataNodesBestEffort( + final boolean includeConsensusQueueReadiness) { final Map<Integer, TDataNodeLocation> dataNodeLocationMap = configManager.getNodeManager().getRegisteredDataNodeLocations(); - final TPullCommitProgressReq request = new TPullCommitProgressReq(); + final TPullCommitProgressReq request = + new TPullCommitProgressReq() + .setIncludeConsensusQueueReadiness(includeConsensusQueueReadiness); final DataNodeAsyncRequestContext<TPullCommitProgressReq, TPullCommitProgressResp> clientHandler = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index ec97a2ff963..1069eb2b7b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -254,6 +254,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateResp; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceResp; import org.apache.iotdb.mpp.rpc.thrift.TCleanDataNodeCacheReq; +import org.apache.iotdb.mpp.rpc.thrift.TConsensusSubscriptionQueueReadiness; import org.apache.iotdb.mpp.rpc.thrift.TConstructSchemaBlackListReq; import org.apache.iotdb.mpp.rpc.thrift.TConstructSchemaBlackListWithTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TConstructViewSchemaBlackListReq; @@ -1736,9 +1737,22 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface SubscriptionAgent.broker().collectAllRegionCommitProgress(dataNodeId); final Map<String, ByteBuffer> subscriptionProgress = SubscriptionAgent.broker().collectAllProgressSnapshots(); - return new TPullCommitProgressResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())) - .setCommitRegionProgress(regionProgress) - .setSubscriptionProgress(subscriptionProgress); + final TPullCommitProgressResp response = + new TPullCommitProgressResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())) + .setCommitRegionProgress(regionProgress) + .setSubscriptionProgress(subscriptionProgress); + if (req.isSetIncludeConsensusQueueReadiness() && req.isIncludeConsensusQueueReadiness()) { + final List<TConsensusSubscriptionQueueReadiness> consensusQueueReadiness = + new ArrayList<>(); + for (final ConsensusSubscriptionSetupHandler.QueueReadiness readiness : + ConsensusSubscriptionSetupHandler.getQueueReadiness()) { + consensusQueueReadiness.add( + new TConsensusSubscriptionQueueReadiness( + readiness.topicName(), readiness.consumerGroupId(), readiness.tableModel())); + } + response.setConsensusQueueReadiness(consensusQueueReadiness); + } + return response; } catch (Exception e) { LOGGER.warn( DataNodeMiscMessages.MISC_LOG_ERROR_OCCURRED_WHEN_PULLING_COMMIT_PROGRESS_48C12E4B, e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTask.java index 15d5f6673f8..ed83a7d4cc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTask.java @@ -141,6 +141,12 @@ public class ShowSubscriptionsTask implements IConfigTask { progressInfo.isSetReadyOnDns() ? progressInfo.getReadyOnDns().toString() : Collections.emptyList().toString()); + writeText( + builder, + 21, + progressInfo.isSetQueuesBoundOnDns() + ? progressInfo.getQueuesBoundOnDns().toString() + : Collections.emptyList().toString()); builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java index 654030eaa2b..4b79ceaabdc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java @@ -79,6 +79,9 @@ public class ConsensusSubscriptionSetupHandler { private static final long FIRST_CONSENSUS_SEARCH_INDEX = 1L; + /** A point-in-time result that all matching local Region queues are bound for a subscription. */ + public record QueueReadiness(String topicName, String consumerGroupId, boolean tableModel) {} + /** Last-known preferred writer node ID per region, used to detect routing changes. */ private static final ConcurrentHashMap<TConsensusGroupId, Integer> lastKnownPreferredWriter = new ConcurrentHashMap<>(); @@ -389,6 +392,102 @@ public class ConsensusSubscriptionSetupHandler { } } + public static Set<QueueReadiness> getQueueReadiness() { + final Set<QueueReadiness> readySubscriptions = new LinkedHashSet<>(); + for (final String consumerGroupId : + SubscriptionAgent.consumer().getAllSubscriptions().keySet()) { + SubscriptionAgent.consumer() + .executeWithConsumerGroupLock( + consumerGroupId, + () -> { + final boolean isTableModel = + SubscriptionAgent.consumer().isTableModel(consumerGroupId); + for (final String topicName : + SubscriptionAgent.consumer() + .getTopicNamesSubscribedByConsumerGroup(consumerGroupId)) { + if (!isConsensusBasedTopic(topicName, isTableModel)) { + continue; + } + try { + if (!areAllLocalQueuesBound(consumerGroupId, topicName, isTableModel)) { + continue; + } + readySubscriptions.add( + new QueueReadiness(topicName, consumerGroupId, isTableModel)); + } catch (final RuntimeException ignored) { + // An uncertain readiness probe must not suppress the normal progress snapshot. + } + } + }); + } + return readySubscriptions; + } + + static boolean isQueueBindingComplete( + final Set<String> matchingRegionIds, + final Set<String> boundRegionIds, + final boolean hasPendingPeerCreationCallback, + final boolean regionSetStable) { + return !hasPendingPeerCreationCallback + && regionSetStable + && boundRegionIds.containsAll(matchingRegionIds); + } + + private static boolean areAllLocalQueuesBound( + final String consumerGroupId, final String topicName, final boolean isTableModel) { + final IConsensus dataRegionConsensus = DataRegionConsensusImpl.getInstance(); + if (!(dataRegionConsensus instanceof IoTConsensus)) { + return false; + } + + final Map<String, TopicConfig> topicConfigs = + SubscriptionAgent.topic().getTopicConfigs(Collections.singleton(topicName), isTableModel); + final TopicConfig topicConfig = topicConfigs.get(topicName); + if (topicConfig == null) { + return false; + } + + final IoTConsensus ioTConsensus = (IoTConsensus) dataRegionConsensus; + final Set<String> regionIdsBeforeCheck = getLocalDataRegionIds(ioTConsensus); + final Set<String> matchingRegionIds = new LinkedHashSet<>(); + boolean hasPendingPeerCreationCallback = false; + for (final ConsensusGroupId groupId : ioTConsensus.getAllConsensusGroupIds()) { + if (!(groupId instanceof DataRegionId)) { + continue; + } + final IoTConsensusServerImpl serverImpl = ioTConsensus.getImpl(groupId); + final DataRegion dataRegion = + StorageEngine.getInstance().getDataRegion((DataRegionId) groupId); + if (serverImpl == null || dataRegion == null) { + return false; + } + if (matchesTopicDataRegion(dataRegion.getDatabaseName(), topicConfig, isTableModel)) { + // A pending callback may still be responsible for binding this matching queue. + hasPendingPeerCreationCallback |= ioTConsensus.isPeerCreationCallbackPending(groupId); + matchingRegionIds.add(groupId.toString()); + } + } + + final Set<String> boundRegionIds = + SubscriptionAgent.broker() + .getConsensusQueueRegionIds(consumerGroupId, Collections.singleton(topicName)) + .getOrDefault(topicName, Collections.emptySet()); + final boolean regionSetStable = + regionIdsBeforeCheck.equals(getLocalDataRegionIds(ioTConsensus)); + return isQueueBindingComplete( + matchingRegionIds, boundRegionIds, hasPendingPeerCreationCallback, regionSetStable); + } + + private static Set<String> getLocalDataRegionIds(final IoTConsensus ioTConsensus) { + final Set<String> regionIds = new LinkedHashSet<>(); + for (final ConsensusGroupId groupId : ioTConsensus.getAllConsensusGroupIds()) { + if (groupId instanceof DataRegionId) { + regionIds.add(groupId.toString()); + } + } + return regionIds; + } + private static boolean isConsensusBasedTopicRequired( final String topicName, final boolean isTableModel) { final String topicMode = SubscriptionAgent.topic().getTopicMode(topicName, isTableModel); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTaskTest.java index e3235c77e14..4b5385778c8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTaskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/subscription/ShowSubscriptionsTaskTest.java @@ -39,25 +39,26 @@ public class ShowSubscriptionsTaskTest { public void testBuildDetailsTSBlock() throws Exception { final TSubscriptionProgressInfo progressInfo = new TSubscriptionProgressInfo( - "topic", - "group", - "DataRegion[1]", - 3, - true, - true, - "CATCHING_UP", - 5L, - 20L, - 6L, - 2L, - 1L, - 2L, - 100L, - 81L, - 123L, - 120L, - "consumer", - 4L); + "topic", + "group", + "DataRegion[1]", + 3, + true, + true, + "CATCHING_UP", + 5L, + 20L, + 6L, + 2L, + 1L, + 2L, + 100L, + 81L, + 123L, + 120L, + "consumer", + 4L) + .setQueuesBoundOnDns(Collections.singletonList(3)); final SettableFuture<ConfigTaskResult> future = SettableFuture.create(); ShowSubscriptionsTask.buildDetailsTSBlock(Collections.singletonList(progressInfo), future); @@ -71,6 +72,9 @@ public class ShowSubscriptionsTaskTest { ColumnHeaderConstant.SEEK_GENERATION, result.getResultSetHeader().getRespColumns().get(19)); assertEquals( ColumnHeaderConstant.READY_ON_DNS, result.getResultSetHeader().getRespColumns().get(20)); + assertEquals( + ColumnHeaderConstant.QUEUES_BOUND_ON_DNS, + result.getResultSetHeader().getRespColumns().get(21)); assertEquals(1, resultSet.getPositionCount()); assertEquals("topic_group", resultSet.getColumn(0).getBinary(0).toString()); assertEquals("CATCHING_UP", resultSet.getColumn(5).getBinary(0).toString()); @@ -79,5 +83,6 @@ public class ShowSubscriptionsTaskTest { assertEquals(20L, resultSet.getColumn(9).getLong(0)); assertEquals(4L, resultSet.getColumn(19).getLong(0)); assertEquals("[]", resultSet.getColumn(20).getBinary(0).toString()); + assertEquals("[3]", resultSet.getColumn(21).getBinary(0).toString()); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java index 19f944b52d0..17e8fbd19e2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java @@ -93,6 +93,29 @@ public class ConsensusSubscriptionSetupHandlerTest { ConsensusSubscriptionSetupHandler.onRegionRouteChanged(null, 0); } + @Test + public void testQueueBindingReadinessRequiresEveryStableRegionAndCompletedCallbacks() { + final Set<String> matchingRegionIds = new LinkedHashSet<>(Arrays.asList("1", "2")); + final Set<String> allBoundRegionIds = new LinkedHashSet<>(Arrays.asList("1", "2")); + final Set<String> partiallyBoundRegionIds = Collections.singleton("1"); + + assertTrue( + ConsensusSubscriptionSetupHandler.isQueueBindingComplete( + matchingRegionIds, allBoundRegionIds, false, true)); + assertFalse( + ConsensusSubscriptionSetupHandler.isQueueBindingComplete( + matchingRegionIds, partiallyBoundRegionIds, false, true)); + assertFalse( + ConsensusSubscriptionSetupHandler.isQueueBindingComplete( + matchingRegionIds, allBoundRegionIds, true, true)); + assertFalse( + ConsensusSubscriptionSetupHandler.isQueueBindingComplete( + matchingRegionIds, allBoundRegionIds, false, false)); + assertTrue( + ConsensusSubscriptionSetupHandler.isQueueBindingComplete( + Collections.emptySet(), Collections.emptySet(), false, true)); + } + @Test public void testSingleTopicSetupFailurePropagates() { SubscriptionException failure = null; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index d439b356094..92f81eac3fd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -157,6 +157,7 @@ public class ColumnHeaderConstant { public static final String LAST_CONSUMER_ID = "LastConsumerId"; public static final String SEEK_GENERATION = "SeekGeneration"; public static final String READY_ON_DNS = "ready_on_dns"; + public static final String QUEUES_BOUND_ON_DNS = "queues_bound_on_dns"; // show cluster status public static final String NODE_TYPE_CONFIG_NODE = "ConfigNode"; @@ -740,7 +741,8 @@ public class ColumnHeaderConstant { new ColumnHeader(LAST_POLL_TIME, TSDataType.INT64), new ColumnHeader(LAST_CONSUMER_ID, TSDataType.TEXT), new ColumnHeader(SEEK_GENERATION, TSDataType.INT64), - new ColumnHeader(READY_ON_DNS, TSDataType.TEXT)); + new ColumnHeader(READY_ON_DNS, TSDataType.TEXT), + new ColumnHeader(QUEUES_BOUND_ON_DNS, TSDataType.TEXT)); public static final List<ColumnHeader> selectIntoColumnHeaders = ImmutableList.of( diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 41da9c53b00..643bff4b22a 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1116,6 +1116,7 @@ struct TSubscriptionProgressInfo { 18: required string lastConsumerId 19: required i64 seekGeneration 20: optional list<i32> readyOnDns + 21: optional list<i32> queuesBoundOnDns // DataNodes reporting all matching local queues bound. } struct TDropSubscriptionReq { diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index aaf76ff7341..795fb2deb9d 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -673,12 +673,21 @@ struct TPushConsumerGroupMetaRespExceptionMessage { } struct TPullCommitProgressReq { + 1: optional bool includeConsensusQueueReadiness // Request a point-in-time queue binding snapshot. +} + +// One entry means all matching local DataRegion queues for this subscription are bound. +struct TConsensusSubscriptionQueueReadiness { + 1: required string topicName + 2: required string consumerGroupId + 3: required bool tableModel } struct TPullCommitProgressResp { 1: required common.TSStatus status 2: optional map<string, binary> commitRegionProgress 3: optional map<string, binary> subscriptionProgress + 4: optional list<TConsensusSubscriptionQueueReadiness> consensusQueueReadiness } struct TSyncSubscriptionProgressReq {
