This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a change to branch fix/subscription-stall-observability
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from e3dc70598be Add subscription delivery stall and memory overcommit
diagnostics
add 1a834815784 Fix read lock leak in disk usage statistics (#18807)
add 94f99d6516a Extend the node state machine and persist node statuses
(#18801)
add cb0269d9fde Standardize ConfigRegionStateMachine log messages (#18811)
add 99a3e06c76b [Fix] Preserve absent local search index during
subscription WAL replay (#18805)
add 83f6428629d Fix subscription WAL replay cursor boundaries (#18810)
add 24fb54f7e77 Bump Thrift to 0.25.0 (#18793)
add 21d53dd015c Bound subscription memory borrowing and report admission
failures (#18812)
add b94aa09ab66 [Fix] Prevent consensus subscription teardown from
blocking unsubscribe (#18804)
add 054f5dedae5 [Subscription] Report retained WAL file gaps and retry
transient reads (#18818)
add f16e5b36fb5 Eliminate OPC UA sink test port allocation race (#18817)
add 0d34a1a7998 Merge master into fix/subscription-stall-observability
add a71f82c9f5f Add debug tracing for consensus subscription prefetch
rounds
No new revisions were added by this update.
Summary of changes:
.github/workflows/multi-language-client.yml | 12 +-
LICENSE-binary | 2 +-
.../org/apache/iotdb/it/env/cluster/EnvUtils.java | 12 +
.../it/cluster/IoTDBClusterNodeErrorStartUpIT.java | 31 +-
.../it/cluster/IoTDBClusterNodeShutdownHookIT.java | 60 +-
.../it/cluster/IoTDBNodeStatusFailoverIT.java | 743 +++++++++++++++++
.../it/cluster/IoTDBNodeStatusPersistenceIT.java | 379 +++++++++
.../confignode/it/cluster/NodeStatusTestAgent.java | 203 +++++
.../load/IoTDBRegionGroupLeaderDistributionIT.java | 3 +-
.../it/partition/IoTDBPartitionCreationIT.java | 58 +-
.../it/partition/IoTDBPartitionDurableIT.java | 49 +-
.../iotdb/pipe/it/single/IoTDBShowReceiversIT.java | 6 +-
.../it/db/it/IoTDBSetSystemStatusTableIT.java | 70 +-
...ableAggregationQueryWithNetworkPartitionIT.java | 5 +-
.../IoTDBTableViewQueryWithCachedDeviceIT.java | 3 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 1 +
.../iotdb/session/it/IoTDBConnectionsIT.java | 53 +-
.../iotdb/session/it/pool/SessionPoolIT.java | 21 +-
...TDBConsensusSubscriptionUnsubscribeTableIT.java | 253 ++++++
.../en/org/apache/iotdb/rpc/i18n/RpcMessages.java | 3 +
.../zh/org/apache/iotdb/rpc/i18n/RpcMessages.java | 3 +
.../org/apache/iotdb/rpc/NonOpenTransport.java | 6 +
.../rpc/TCompressedElasticFramedTransport.java | 4 +
.../apache/iotdb/rpc/TElasticFramedTransport.java | 32 +
.../iotdb/rpc/TElasticFramedTransportTest.java | 101 +++
.../base/AbstractSubscriptionConsumer.java | 18 +-
.../base/SubscriptionConsumerLifecycleTest.java | 50 ++
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 65 +-
.../iotdb/confignode/i18n/ManagerMessages.java | 8 +-
.../iotdb/confignode/i18n/ProcedureMessages.java | 7 +
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 68 +-
.../iotdb/confignode/i18n/ManagerMessages.java | 8 +-
.../iotdb/confignode/i18n/ProcedureMessages.java | 7 +
.../handlers/heartbeat/AINodeHeartbeatHandler.java | 4 +-
.../heartbeat/ConfigNodeHeartbeatHandler.java | 4 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 7 +-
.../consensus/request/ConfigPhysicalPlan.java | 4 +
.../consensus/request/ConfigPhysicalPlanType.java | 2 +
...sionInfoPlan.java => UpdateNodeStatusPlan.java} | 57 +-
.../statemachine/ConfigRegionStateMachine.java | 16 +-
.../iotdb/confignode/manager/ConfigManager.java | 54 +-
.../apache/iotdb/confignode/manager/IManager.java | 18 +-
.../iotdb/confignode/manager/ProcedureManager.java | 13 +-
.../confignode/manager/RetryFailedTasksThread.java | 3 +-
.../iotdb/confignode/manager/load/LoadManager.java | 76 +-
.../manager/load/balancer/RegionBalancer.java | 12 +-
.../manager/load/cache/AbstractLoadCache.java | 9 +-
.../confignode/manager/load/cache/LoadCache.java | 140 +++-
.../load/cache/consensus/ConsensusGroupCache.java | 3 +-
.../load/cache/node/AINodeHeartbeatCache.java | 48 --
.../manager/load/cache/node/BaseNodeCache.java | 84 +-
.../load/cache/node/ConfigNodeHeartbeatCache.java | 39 +-
.../load/cache/node/DataNodeHeartbeatCache.java | 57 +-
.../load/cache/node/NodeHeartbeatSample.java | 7 +-
.../manager/load/cache/node/NodeStatistics.java | 17 +
.../manager/load/cache/region/RegionCache.java | 3 +-
.../load/cache/region/RegionGroupCache.java | 2 +-
.../manager/load/service/StatisticsService.java | 2 +-
.../iotdb/confignode/manager/node/NodeManager.java | 25 +-
.../runtime/SubscriptionRuntimeCoordinator.java | 7 +-
.../persistence/executor/ConfigPlanExecutor.java | 3 +
.../confignode/persistence/node/NodeInfo.java | 115 ++-
.../procedure/env/ConfigNodeProcedureEnv.java | 7 +-
.../procedure/env/RegionMaintainHandler.java | 2 +-
.../procedure/env/RemoveDataNodeHandler.java | 71 +-
.../procedure/impl/node/RemoveAINodeProcedure.java | 1 +
.../impl/node/RemoveDataNodesProcedure.java | 6 +-
.../procedure/impl/schema/SchemaUtils.java | 9 +-
.../confignode/service/ConfigNodeShutdownHook.java | 162 +++-
.../heartbeat/DataNodeHeartbeatHandlerTest.java | 100 +++
.../request/ConfigPhysicalPlanSerDeTest.java | 67 ++
.../manager/ConfigManagerShutdownReportTest.java | 161 ++++
.../confignode/manager/ProcedureManagerTest.java | 25 +
.../manager/RetryFailedTasksThreadTest.java | 107 +++
.../confignode/manager/load/LoadManagerTest.java | 115 ++-
.../manager/load/balancer/RegionBalancerTest.java | 144 ++++
.../router/priority/GreedyPriorityTest.java | 2 +-
.../priority/LeaderPriorityBalancerTest.java | 2 +-
.../load/cache/ConsensusGroupCacheTest.java | 2 +-
.../load/cache/LoadCacheNodeStatusTest.java | 336 ++++++++
.../cache/LoadCachePersistedNodeStatusTest.java | 898 +++++++++++++++++++++
.../manager/load/cache/NodeCacheTest.java | 228 +++++-
.../manager/node/NodeManagerStatusReasonTest.java | 133 +++
.../SubscriptionRuntimeCoordinatorTest.java | 119 +++
.../iotdb/confignode/persistence/NodeInfoTest.java | 269 ++++++
.../procedure/env/ConfigNodeProcedureEnvTest.java | 93 +++
.../procedure/env/RegionMaintainHandlerTest.java | 68 ++
.../impl/node/RemoveDataNodePersistenceTest.java | 581 +++++++++++++
.../procedure/impl/schema/SchemaUtilsTest.java | 91 +++
.../service/ConfigNodeShutdownHookTest.java | 346 ++++++++
.../iotdb/consensus/i18n/IoTConsensusMessages.java | 4 +
.../iotdb/consensus/i18n/IoTConsensusMessages.java | 4 +
.../subscription/SubscriptionQueueAdmission.java | 18 +-
.../subscription/SubscriptionQueueRegistry.java | 39 +-
.../SubscriptionQueueRejectionReason.java | 42 +
iotdb-core/datanode/pom.xml | 16 +
.../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 3 +-
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 59 +-
.../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 2 +-
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 58 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 35 +-
.../InformationSchemaContentSupplierFactory.java | 31 +-
.../config/executor/ClusterConfigTaskExecutor.java | 8 +-
.../config/metadata/ShowClusterDetailsTask.java | 63 +-
.../execution/config/metadata/ShowClusterTask.java | 40 +-
.../config/metadata/ShowConfigNodesTask.java | 28 +-
.../config/metadata/ShowDataNodesTask.java | 30 +-
.../iotdb/db/service/DataNodeShutdownHook.java | 4 +-
.../db/storageengine/dataregion/DataRegion.java | 6 +-
.../dataregion/memtable/TsFileProcessor.java | 4 +-
.../dataregion/utils/DiskUsageStatisticUtil.java | 12 +-
.../dataregion/wal/buffer/WALBuffer.java | 10 +-
.../wal/checkpoint/CheckpointManager.java | 4 +-
.../broker/ConsensusSubscriptionBroker.java | 9 +-
.../consensus/ConsensusPrefetchingQueue.java | 573 ++++++++++---
.../broker/consensus/ProgressWALIterator.java | 264 ++++--
...nsensusSubscriptionPrefetchingQueueMetrics.java | 110 +++
.../resource/SubscriptionMemoryManager.java | 333 +++++++-
.../ConsensusSubscriptionPrefetchExecutor.java | 10 +
.../task/subtask/ConsensusPrefetchSubtask.java | 12 +
.../DataNodeInternalRPCServiceImplDiskTest.java | 43 +-
...eInternalRPCServiceImplSetSystemStatusTest.java | 91 +++
.../metadata/ShowClusterDetailsTaskTest.java | 172 ++++
.../config/metadata/ShowClusterTaskTest.java | 168 ++++
.../config/metadata/ShowConfigNodesTaskTest.java | 121 +++
.../config/metadata/ShowDataNodesTaskTest.java | 114 +++
.../dataregion/utils/TableDiskUsageTest.java | 8 +
.../wal/buffer/WALBufferRollRecoveryTest.java | 4 +-
.../rescon/disk/FolderManagerTest.java | 57 +-
...onsensusPrefetchingQueueDataNodeMemoryTest.java | 261 +++++-
.../ConsensusPrefetchingQueueSeekTest.java | 241 ++++++
.../consensus/ConsensusPrefetchingQueueTest.java | 383 ++++++++-
...nsensusPrefetchingQueueWalBackpressureTest.java | 19 +-
.../broker/consensus/ProgressWALIteratorTest.java | 283 ++++++-
...susSubscriptionPrefetchingQueueMetricsTest.java | 88 ++
.../resource/SubscriptionMemoryManagerTest.java | 110 +++
.../conf/iotdb-system.properties.template | 29 +
.../apache/iotdb/commons/cluster/NodeStatus.java | 102 ++-
.../apache/iotdb/commons/cluster/RegionStatus.java | 5 +
.../apache/iotdb/commons/conf/CommonConfig.java | 57 +-
.../apache/iotdb/commons/disk/FolderManager.java | 14 +-
.../commons/disk/strategy/DirectoryStrategy.java | 4 +-
.../commons/enums/HandleSystemErrorStrategy.java | 6 +-
.../schema/column/ColumnHeaderConstant.java | 2 +
.../commons/schema/table/InformationSchema.java | 3 +
.../iotdb/commons/service/metric/enums/Metric.java | 10 +
.../iotdb/commons/service/metric/enums/Tag.java | 1 +
.../iotdb/commons/cluster/NodeStatusTest.java | 91 +++
.../iotdb/commons/conf/CommonConfigTest.java | 222 ++++-
.../src/main/thrift/confignode.thrift | 21 +-
.../pipe/plugin/sink/opcua/OpcUaSinkTest.java | 79 +-
pom.xml | 4 +-
152 files changed, 10920 insertions(+), 1044 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionUnsubscribeTableIT.java
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/{UpdateVersionInfoPlan.java
=> UpdateNodeStatusPlan.java} (56%)
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandlerTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerShutdownReportTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThreadTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancerTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerStatusReasonTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinatorTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnvTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtilsTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java
copy
external-service-impl/mqtt/src/main/java/org/apache/iotdb/mqtt/Message.java =>
iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/subscription/SubscriptionQueueAdmission.java
(71%)
create mode 100644
iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/subscription/SubscriptionQueueRejectionReason.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplSetSystemStatusTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTaskTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTaskTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTaskTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTaskTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/cluster/NodeStatusTest.java
