This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a change to branch fix/subscription-wal-backed-inflight
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 7e1ce243f83 [Subscription] Drop eager WAL replayability checks
add a498d7d2a46 Fix schema mutation validation, accounting and deletion
performance (#18861)
add edd5a74c42d Prevent query with explict limit 0 (#18858)
add 0c0116afe22 Bound consensus subscription WAL scans per prefetch round
(#18849)
add 6eca5b841d7 [Pipe] Add SHOW PIPE MEMORY for named memory blocks
(#18575)
add 8effe5d6ce4 Support retaining subscription progress after unsubscribe
(#18718)
add db8bc17a27f [Feature] Expose DROP TABLE mod file progress (#18855)
add 5842fbc6a7b [Pipe] Add reliable DataNode completion metric (#18563)
add 3f1706cbce0 Fix subscription realtime retention under shared memory
pressure (#18870)
add f106a0f1d16 [Test] Wait for asynchronous table deletion in session IT
(#18877)
add 1b290ae320c [Bugfix] Reclaim RPC buffers by evicting idle clients
(#18853)
add ed61790920d [Test] Fix pipe memory ownership test compilation (#18881)
add f8135bb7284 [Test] Fix Ratis mini-cluster shutdown race (#18879)
add e3bbab5e7e8 [Subscription] Support per-topic uncommitted event recycle
timeouts (#18876)
add e1baa97f2db [Fix] Stop only the listening DataNode process in Windows
stop-datanode.bat (#18883)
add d77a81534e3 Fix deletion WAL logging and flush completion ordering
(#18872)
add 673434b3335 Fix quoted tree path parsing in table views (#18884)
add 264053f1423 [Test] Fix deletion regression test flush control
add 532d6c13add Fence IoTConsensus applies before storage shutdown (#18841)
add 6402cd9594b Revert "[Test] Fix deletion regression test flush control"
add 091c16a16d1 Merge branch 'master' of https://github.com/apache/iotdb
into fix/subscription-wal-backed-inflight
No new revisions were added by this update.
Summary of changes:
.../apache/iotdb/db/it/IoTDBShowPipeMemoryIT.java | 246 ++++++
.../manual/enhanced/IoTDBPipeClusterIT.java | 41 +-
.../auto/enhanced/IoTDBPipeClusterIT.java | 38 +-
.../it/query/old/IoTDBSimpleQueryTableIT.java | 24 +
.../query/view/recent/IoTDBTableViewQueryIT.java | 2 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 67 +-
.../iotdb/relational/it/schema/IoTDBDeviceIT.java | 13 +-
.../it/schema/IoTDBDropTableModProgressIT.java | 105 +++
.../iotdb/relational/it/schema/IoTDBTableIT.java | 34 +-
.../it/session/IoTDBSessionRelationalIT.java | 24 +-
.../org/apache/iotdb/pipe/api/event/Event.java | 18 +-
.../iotdb/rpc/subscription/config/TopicConfig.java | 19 +
.../rpc/subscription/config/TopicConstant.java | 7 +
.../rpc/subscription/config/TopicConfigTest.java | 34 +
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 6 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 9 +
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 9 +
.../client/async/CnToDnAsyncRequestType.java | 1 +
.../CnToDnInternalServiceAsyncRequestManager.java | 7 +
.../rpc/DataNodeAsyncRequestRPCHandler.java | 9 +
...java => GetDropTableModProgressRPCHandler.java} | 37 +-
.../iotdb/confignode/manager/ProcedureManager.java | 11 +
.../persistence/subscription/SubscriptionInfo.java | 137 +++-
.../impl/schema/table/DropTableProcedure.java | 178 ++++-
.../subscription/SubscriptionInfoTest.java | 125 ++++
.../SubscriptionInfoTopicValidationTest.java | 150 ++++
.../impl/schema/table/DropTableProcedureTest.java | 156 ++++
.../apache/iotdb/consensus/iot/IoTConsensus.java | 15 +-
.../consensus/iot/IoTConsensusServerImpl.java | 47 +-
.../subscription/SubscriptionQueueRegistry.java | 50 +-
.../consensus/iot/IoTConsensusServerImplTest.java | 94 +++
.../SubscriptionQueueRegistryTest.java | 42 ++
.../iotdb/consensus/ratis/RecoverReadTest.java | 70 ++
.../apache/iotdb/consensus/ratis/TestUtils.java | 51 +-
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 2 +
.../iotdb/db/i18n/DataNodeQueryMessages.java | 5 +-
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 2 +
.../iotdb/db/i18n/DataNodeQueryMessages.java | 5 +-
.../dataregion/DataRegionStateMachine.java | 6 +-
.../IoTConsensusDataRegionStateMachine.java | 7 +
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 7 +-
.../iotdb/db/pipe/agent/task/PipeDataNodeTask.java | 6 +
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 87 +++
.../task/builder/PipeDataNodeTaskBuilder.java | 1 +
.../agent/task/connection/PipeEventCollector.java | 23 +-
.../agent/task/stage/PipeTaskProcessorStage.java | 2 +
.../pipe/agent/task/stage/PipeTaskSourceStage.java | 18 +
.../subtask/processor/PipeProcessorSubtask.java | 10 +-
.../sink/PipeRealtimePriorityBlockingQueue.java | 4 +-
.../db/pipe/event/UserDefinedEnrichedEvent.java | 11 +
.../db/pipe/event/common/PipeInsertionEvent.java | 9 +
.../event/common/heartbeat/PipeHeartbeatEvent.java | 102 ++-
.../statement/PipeStatementInsertionEvent.java | 15 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 32 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 22 +-
.../event/common/terminate/PipeTerminateEvent.java | 5 +
.../common/tsfile/PipeTsFileInsertionEvent.java | 41 +-
.../tsfile/parser/TsFileInsertionEventParser.java | 19 +-
.../TsFileInsertionEventParserMemoryManager.java | 68 +-
.../query/TsFileInsertionEventQueryParser.java | 9 +-
.../scan/TsFileInsertionEventScanParser.java | 15 +-
.../table/TsFileInsertionEventTableParser.java | 19 +-
.../event/realtime/PipeRealtimeEventFactory.java | 4 +
.../overview/PipeDataNodeCompletionOperator.java | 420 +++++++++++
.../PipeDataNodeRemainingEventAndTimeOperator.java | 4 +
.../overview/PipeDataNodeSinglePipeMetrics.java | 126 +++-
.../downsampling/DownSamplingProcessor.java | 3 +
.../downsampling/PartialPathLastObjectCache.java | 13 +-
.../protocol/thrift/IoTDBDataNodeReceiver.java | 12 +-
.../pipe/resource/memory/PipeFixedMemoryBlock.java | 37 +-
.../db/pipe/resource/memory/PipeMemoryBlock.java | 282 ++++++-
...BlockType.java => PipeMemoryBlockCategory.java} | 34 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 821 +++++++++++++++++----
.../resource/memory/PipeModelFixedMemoryBlock.java | 27 +-
.../resource/memory/PipeTabletMemoryBlock.java | 14 +-
.../resource/memory/PipeTsFileMemoryBlock.java | 14 +-
.../resource/tsfile/PipeTsFilePublicResource.java | 29 +-
.../IoTDBDataNodeCacheLeaderClientManager.java | 8 +-
.../batch/PipeSchemaRegionWritePlanEventBatch.java | 9 +-
.../evolvable/batch/PipeTabletEventBatch.java | 9 +-
.../protocol/airgap/IoTDBDataRegionAirGapSink.java | 7 +-
.../iotconsensusv2/IoTConsensusV2SyncSink.java | 7 +-
.../IoTConsensusV2TsFileInsertionEventHandler.java | 8 +-
.../IoTConsensusV2TransferBatchReqBuilder.java | 9 +-
.../sink/protocol/legacy/IoTDBLegacyPipeSink.java | 7 +-
.../async/handler/PipeTransferTsFileHandler.java | 8 +-
.../thrift/sync/IoTDBDataRegionSyncSink.java | 7 +-
.../source/dataregion/IoTDBDataRegionSource.java | 15 +
.../PipeRealtimeDataRegionHybridSource.java | 20 +-
.../realtime/PipeRealtimeDataRegionLogSource.java | 1 +
.../realtime/PipeRealtimeDataRegionSource.java | 41 +-
.../PipeRealtimeDataRegionTsFileSource.java | 1 +
.../realtime/assigner/DisruptorQueue.java | 8 +-
.../realtime/assigner/PipeDataRegionAssigner.java | 243 +++++-
.../listener/PipeInsertionDataNodeListener.java | 77 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 28 +-
.../schema/source/DeviceBlackListConstructor.java | 14 +-
.../InformationSchemaContentSupplierFactory.java | 67 ++
.../iotdb/db/queryengine/plan/Coordinator.java | 23 +
.../db/queryengine/plan/parser/ASTVisitor.java | 6 +
.../relational/analyzer/StatementAnalyzer.java | 4 +-
.../DataNodeLocationSupplierFactory.java | 1 +
.../security/TreeAccessCheckVisitor.java | 10 +
.../plan/relational/sql/parser/AstBuilder.java | 30 +-
.../queryengine/plan/statement/StatementType.java | 1 +
.../plan/statement/StatementVisitor.java | 5 +
.../ShowPipeMemoryStatement.java} | 41 +-
.../attribute/DeviceAttributeStore.java | 75 +-
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 23 +-
.../impl/mem/mnode/impl/BasicInternalMNode.java | 60 +-
.../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 1 +
.../schemaregion/utils/MetaFormatUtils.java | 23 +
.../java/org/apache/iotdb/db/service/DataNode.java | 4 +
.../iotdb/db/service/DataNodeShutdownHook.java | 3 +
.../iotdb/db/storageengine/StorageEngine.java | 30 +-
.../db/storageengine/dataregion/DataRegion.java | 214 +++++-
.../dataregion/memtable/TsFileProcessor.java | 115 ++-
.../modification/DropTableModProgress.java | 65 ++
.../load/memory/LoadTsFileParserMemoryManager.java | 5 +-
.../agent/SubscriptionBrokerAgent.java | 22 +-
.../agent/SubscriptionConsumerAgent.java | 42 ++
.../agent/SubscriptionRuntimeAgent.java | 2 +
.../subscription/agent/SubscriptionTopicAgent.java | 52 ++
.../broker/ConsensusSubscriptionBroker.java | 32 +-
.../broker/SubscriptionPrefetchingQueue.java | 13 +-
.../consensus/ConsensusPrefetchingQueue.java | 361 ++++++---
.../ConsensusSubscriptionCommitManager.java | 76 ++
.../ConsensusSubscriptionSetupHandler.java | 308 +++++++-
.../ConsensusSubscriptionWalRetention.java | 163 ++++
.../broker/consensus/ProgressWALIterator.java | 78 +-
.../db/subscription/event/SubscriptionEvent.java | 14 +-
.../event/cache/SubscriptionPollResponseCache.java | 8 +-
.../response/SubscriptionEventTabletResponse.java | 8 +-
.../response/SubscriptionEventTsFileResponse.java | 8 +-
.../dataregion/IoTConsensusShutdownTest.java | 593 +++++++++++++++
.../schemaRegion/SchemaRegionAliasAndTagTest.java | 62 ++
.../task/connection/PipeEventCollectorTest.java | 4 +-
.../processor/PipeProcessorSubtaskTest.java | 87 +++
.../common/heartbeat/PipeHeartbeatEventTest.java | 92 +++
.../PipeDataNodeCompletionOperatorTest.java | 248 +++++++
.../PipeDataNodeSinglePipeMetricsTest.java | 221 ++++++
.../memory/PipeMemoryManagerResizeTest.java | 332 ++++++++-
.../airgap/IoTDBDataRegionAirGapSinkTest.java | 34 +-
.../realtime/PipeRealtimeDataRegionSourceTest.java | 25 +
.../assigner/PipeDataRegionAssignerTest.java | 88 +++
.../PipeInsertionDataNodeListenerTest.java | 66 ++
.../source/DeviceBlackListConstructorTest.java | 120 +++
.../node/write/RelationalDeleteDataNodeTest.java | 22 +
.../plan/relational/analyzer/AnalyzerTest.java | 9 +
.../analyzer/EvaluateEmptyIntersectTest.java | 25 +-
.../analyzer/RemoveEmptyExceptBranchesTest.java | 82 +-
.../relational/analyzer/UnionOptimizeTest.java | 28 +-
...mentTest.java => CreateViewPrefixPathTest.java} | 29 +-
.../relational/sql/parser/ShowPipeMemoryTest.java | 63 ++
.../attribute/DeviceAttributeStoreTest.java | 129 ++++
.../mtree/impl/mem/MTreeBelowSGMemoryImplTest.java | 129 ++++
.../storageengine/dataregion/DataRegionTest.java | 295 +++++++-
.../dataregion/memtable/TsFileProcessorTest.java | 217 ++++++
.../load/memory/LoadTsFileMemoryManagerTest.java | 2 +-
.../agent/SubscriptionTopicAgentTest.java | 116 +++
...riptionPrefetchingQueueRecycleIntervalTest.java | 160 ++++
...onsensusPrefetchingQueueDataNodeMemoryTest.java | 170 ++++-
...onsensusPrefetchingQueueInitializationTest.java | 5 +-
.../ConsensusPrefetchingQueueSeekTest.java | 3 +-
.../consensus/ConsensusPrefetchingQueueTest.java | 331 +++++++--
...nsensusPrefetchingQueueWalBackpressureTest.java | 160 +++-
.../ConsensusSubscriptionCommitStateTest.java | 35 +
.../ConsensusSubscriptionSetupHandlerTest.java | 47 ++
.../ConsensusSubscriptionWalRetentionTest.java | 102 +++
.../broker/consensus/ProgressWALIteratorTest.java | 110 +++
.../conf/iotdb-system.properties.template | 13 +
.../client/property/ClientPoolProperty.java | 5 +-
.../agent/plugin/builtin/BuiltinPipePlugin.java | 46 +-
.../pipe/agent/task/meta/PipeStaticMeta.java | 33 +
.../commons/pipe/agent/task/meta/PipeTaskMeta.java | 17 +-
.../task/progress/PipeEventCommitManager.java | 15 +
.../iotdb/commons/pipe/event/EnrichedEvent.java | 14 +
.../commons/pipe/event/ProgressReportEvent.java | 5 +
.../schema/column/ColumnHeaderConstant.java | 16 +
.../commons/schema/table/InformationSchema.java | 43 ++
.../iotdb/commons/service/metric/enums/Metric.java | 1 +
.../meta/consumer/CommitProgressKeeper.java | 15 +
.../iotdb/commons/pipe/task/PipeMetaDeSerTest.java | 74 ++
.../meta/consumer/CommitProgressKeeperTest.java | 19 +
.../iotdb/rpc/ClientPoolIdleEvictionTest.java | 326 ++++++++
.../db/relational/grammar/sql/RelationalSql.g4 | 8 +-
.../src/main/thrift/confignode.thrift | 2 +
.../src/main/thrift/datanode.thrift | 19 +
scripts/sbin/windows/stop-datanode.bat | 5 +-
191 files changed, 10842 insertions(+), 868 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBShowPipeMemoryIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDropTableModProgressIT.java
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/{SubmitTestConnectionTaskRPCHandler.java
=> GetDropTableModProgressRPCHandler.java} (57%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeCompletionOperator.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/{PipeMemoryBlockType.java
=> PipeMemoryBlockCategory.java} (55%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/{template/ShowSchemaTemplateStatement.java
=> pipe/ShowPipeMemoryStatement.java} (56%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DropTableModProgress.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionWalRetention.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEventTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeCompletionOperatorTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssignerTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructorTest.java
copy
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/{TopicStatementTest.java
=> CreateViewPrefixPathTest.java} (58%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/ShowPipeMemoryTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueueRecycleIntervalTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionWalRetentionTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/rpc/ClientPoolIdleEvictionTest.java
