This is an automated email from the ASF dual-hosted git repository. yongzao pushed a commit to branch addlogicalclockinterface in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5addf4ab76447af8b02e474afbd510de01e06046 Merge: 40328ce5a01 381948b52f5 Author: YongzaoDan <[email protected]> AuthorDate: Fri Apr 12 15:43:40 2024 +0800 Merge branch 'master' into addlogicalclockinterface .github/workflows/pipe-it-2cluster.yml | 8 +- README.md | 62 +- example/pipe-opc-ua-sink/pom.xml | 34 + .../apache/iotdb/SubscriptionSessionExample.java | 18 +- .../iotdb/it/env/cluster/config/MppBaseConfig.java | 4 +- .../it/env/cluster/config/MppConfigNodeConfig.java | 6 + .../iotdb/it/env/cluster/env/AbstractEnv.java | 15 + .../it/env/cluster/node/AbstractNodeWrapper.java | 11 + .../env/remote/config/RemoteConfigNodeConfig.java | 6 + .../iotdb/it/env/remote/env/RemoteServerEnv.java | 10 + .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 4 + .../apache/iotdb/itbase/env/ConfigNodeConfig.java | 2 + ...oTDBMultiDBRegionGroupLeaderDistributionIT.java | 154 +++++ ...a => IoTDBRegionGroupLeaderDistributionIT.java} | 123 ++-- .../IoTDBRegionMigrateReliabilityITFramework.java} | 431 +++++++----- .../CoordinatorRemoveRemotePeerCrashIT.java | 51 ++ .../IoTDBRegionMigrateDataNodeCrashIT.java | 72 ++ .../pass/IoTDBRegionMigrateConfigNodeCrashIT.java | 92 +++ .../pass/IoTDBRegionMigrateNormalIT.java | 30 +- .../pass/IoTDBRegionMigrateOtherIT.java | 45 ++ .../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 67 ++ .../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 3 +- .../org/apache/iotdb/db/it/utils/TestUtils.java | 5 + .../pipe/it/autocreate/AbstractPipeDualAutoIT.java | 4 + .../pipe/it/autocreate/IoTDBPipeClusterIT.java | 75 ++- .../pipe/it/autocreate/IoTDBPipeIdempotentIT.java | 4 + .../pipe/it/autocreate/IoTDBPipeProcessorIT.java | 4 + .../pipe/it/autocreate/IoTDBPipeProtocolIT.java | 12 + .../pipe/it/manual/AbstractPipeDualManualIT.java | 4 + .../pipe/it/manual/IoTDBPipeMetaHistoricalIT.java | 4 + .../it/manual/IoTDBPipeMetaLeaderChangeIT.java | 4 + .../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 2 + .../it/dual/AbstractSubscriptionDualIT.java | 5 + .../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 725 ++++++++++++++------- .../it/dual/IoTDBSubscriptionTopicIT.java | 157 ++++- .../it/local/IoTDBSubscriptionBasicIT.java | 12 +- .../it/local/IoTDBSubscriptionIdempotentIT.java | 22 +- .../it/local/IoTDBSubscriptionRestartIT.java | 298 ++++++++- .../parameter/PipeParameterValidator.java | 22 +- .../rpc/subscription/config/ConsumerConstant.java | 17 +- .../exception}/SubscriptionException.java | 2 +- .../SubscriptionParameterNotValidException.java} | 18 +- .../SubscriptionPollTimeOutException.java | 16 +- .../payload/response/PipeSubscribePollResp.java | 24 +- .../subscription/ConsumerHeartbeatWorker.java | 26 +- .../subscription/PullConsumerAutoCommitWorker.java | 10 +- .../session/subscription/SubscriptionConsumer.java | 383 +++++++++-- .../subscription/SubscriptionEndpointsSyncer.java | 131 ++++ .../session/subscription/SubscriptionProvider.java | 48 +- .../subscription/SubscriptionPullConsumer.java | 99 ++- .../subscription/SubscriptionPushConsumer.java | 2 +- .../session/subscription/SubscriptionSession.java | 20 +- .../SubscriptionSessionConnection.java | 15 - .../session/subscription/model/Subscription.java | 6 +- .../iotdb/session/subscription/model/Topic.java | 4 +- iotdb-connector/grafana-plugin/yarn.lock | 16 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 2 +- .../confignode/conf/ConfigNodeDescriptor.java | 11 +- .../confignode/conf/ConfigNodeRemoveCheck.java | 4 +- .../confignode/conf/ConfigNodeStartupCheck.java | 2 +- .../confignode/conf/SystemPropertiesUtils.java | 7 +- .../statemachine/ConfigRegionStateMachine.java | 23 +- .../iotdb/confignode/manager/ConfigManager.java | 17 +- .../apache/iotdb/confignode/manager/IManager.java | 7 + .../manager/load/balancer/RouteBalancer.java | 3 +- .../router/leader/GreedyLeaderBalancer.java | 129 +--- .../balancer/router/leader/ILeaderBalancer.java | 6 +- .../router/leader/MinCostFlowLeaderBalancer.java | 211 +++--- .../iotdb/confignode/manager/node/NodeManager.java | 7 + .../{transfer => }/agent/PipeConfigNodeAgent.java | 10 +- .../agent/plugin/PipeConfigNodePluginAgent.java | 2 +- .../PipeConfigRegionConnectorConstructor.java | 6 +- .../PipeConfigRegionExtractorConstructor.java | 4 +- .../PipeConfigRegionProcessorConstructor.java | 2 +- .../receiver/IoTDBConfigNodeReceiverAgent.java | 24 +- .../agent/runtime/PipeConfigNodeRuntimeAgent.java | 6 +- .../agent/runtime/PipeConfigRegionListener.java | 6 +- .../agent/task/PipeConfigNodeTaskAgent.java | 12 +- .../client/IoTDBConfigNodeSyncClientManager.java | 14 +- .../PipeTransferConfigNodeHandshakeV1Req.java | 2 +- .../PipeTransferConfigNodeHandshakeV2Req.java | 2 +- .../payload}/PipeTransferConfigPlanReq.java | 2 +- .../PipeTransferConfigSnapshotPieceReq.java | 2 +- .../PipeTransferConfigSnapshotSealReq.java | 2 +- .../IoTDBConfigRegionAirGapConnector.java | 12 +- .../protocol}/IoTDBConfigRegionConnector.java | 16 +- .../runtime/PipeHeartbeatScheduler.java | 2 +- .../execution/PipeConfigNodeSubtask.java | 6 +- .../execution/PipeConfigNodeSubtaskExecutor.java | 2 +- .../extractor/ConfigRegionListeningFilter.java | 13 +- .../extractor/ConfigRegionListeningQueue.java | 2 +- .../extractor/IoTDBConfigRegionExtractor.java | 4 +- .../protocol}/IoTDBConfigNodeReceiver.java | 58 +- .../PipeConfigPhysicalPlanExceptionVisitor.java | 2 +- .../PipeConfigPhysicalPlanTSStatusVisitor.java | 5 +- .../{transfer => }/task/PipeConfigNodeTask.java | 2 +- .../task/PipeConfigNodeTaskBuilder.java | 4 +- .../task/PipeConfigNodeTaskStage.java | 6 +- .../persistence/executor/ConfigPlanExecutor.java | 2 +- .../persistence/partition/PartitionInfo.java | 7 +- .../confignode/persistence/pipe/PipeInfo.java | 2 +- .../persistence/subscription/SubscriptionInfo.java | 2 +- .../impl/region/AddRegionPeerProcedure.java | 4 + .../impl/region/RegionMigrateProcedure.java | 2 + .../impl/region/RemoveRegionPeerProcedure.java | 4 + .../AbstractOperateSubscriptionProcedure.java | 2 +- .../consumer/AlterConsumerGroupProcedure.java | 2 +- .../runtime/ConsumerGroupMetaSyncProcedure.java | 2 +- .../subscription/CreateSubscriptionProcedure.java | 2 +- .../subscription/DropSubscriptionProcedure.java | 2 +- .../subscription/topic/AlterTopicProcedure.java | 2 +- .../subscription/topic/CreateTopicProcedure.java | 2 +- .../subscription/topic/DropTopicProcedure.java | 2 +- .../topic/runtime/TopicMetaSyncProcedure.java | 2 +- .../iotdb/confignode/service/ConfigNode.java | 2 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 8 +- ...alancerTest.java => CFDLeaderBalancerTest.java} | 66 +- .../router/leader/GreedyLeaderBalancerTest.java | 6 +- .../leader/LeaderBalancerComparisonTest.java | 4 +- .../connector/IoTDBConfigRegionConnectorTest.java | 2 +- .../PipeConfigNodeThriftRequestTest.java | 10 +- .../PipeConfigNodeSubtaskExecutorTest.java | 2 - .../extractor/IoTDBConfigRegionExtractorTest.java | 1 - .../ConfigRegionListeningQueueTest.java | 2 +- .../apache/iotdb/consensus/iot/IoTConsensus.java | 58 +- .../consensus/iot/IoTConsensusServerImpl.java | 69 +- .../service/IoTConsensusRPCServiceProcessor.java | 5 + .../iotdb/consensus/ratis/RatisConsensus.java | 5 +- .../iotdb/consensus/ratis/RatisConsensusTest.java | 7 +- .../org/apache/iotdb/db/audit/AuditLogger.java | 2 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 25 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 17 +- .../db/conf/rest/IoTDBRestServiceDescriptor.java | 4 +- .../schemaregion/SchemaRegionStateMachine.java | 16 +- .../db/exception/PartitionViolationException.java | 6 +- .../agent/receiver/PipeDataNodeReceiverAgent.java | 6 +- .../agent/runtime/PipeDataNodeRuntimeAgent.java | 2 +- .../client/IoTDBDataNodeAsyncClientManager.java | 86 ++- .../client/IoTDBDataNodeSyncClientManager.java | 5 +- ...IoTDBThriftSyncPipeTransferBatchReqBuilder.java | 2 +- .../builder/PipeTransferBatchReqBuilder.java | 12 +- .../connector/payload/legacy/DeletionPipeData.java | 4 +- .../db/pipe/connector/payload/legacy/PipeData.java | 2 +- .../connector/payload/legacy/TsFilePipeData.java | 4 +- .../async/IoTDBDataRegionAsyncConnector.java | 4 +- .../thrift/sync/IoTDBDataNodeSyncConnector.java | 5 +- .../protocol/websocket/WebSocketConnector.java | 6 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 4 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 42 +- .../PipeConnectorSubtaskExecutor.java | 2 +- .../PipeProcessorSubtaskExecutor.java | 2 +- .../{executor => }/PipeSubtaskExecutorManager.java | 2 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 2 +- .../realtime/assigner/PipeDataRegionAssigner.java | 4 +- .../iotdb/db/pipe/metric/PipeConnectorMetrics.java | 46 +- .../iotdb/db/pipe/metric/PipeExtractorMetrics.java | 49 +- .../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 46 +- .../{matcher => }/CachedSchemaPatternMatcher.java | 2 +- .../{matcher => }/PipeDataRegionMatcher.java | 2 +- .../processor/aggregate/AggregateProcessor.java | 62 +- .../SimpleConsensusProgressIndexAssigner.java | 2 +- .../{ => protocol}/airgap/IoTDBAirGapReceiver.java | 51 +- .../airgap/IoTDBAirGapReceiverAgent.java | 2 +- .../legacy/IoTDBLegacyPipeReceiverAgent.java | 2 +- .../legacy/loader/DeletionLoader.java | 2 +- .../{ => protocol}/legacy/loader/ILoader.java | 2 +- .../{ => protocol}/legacy/loader/TsFileLoader.java | 2 +- .../thrift/IoTDBDataNodeReceiver.java | 68 +- .../thrift/IoTDBDataNodeReceiverAgent.java | 20 +- .../{ => visitor}/PipePlanToStatementVisitor.java | 2 +- .../PipeStatementExceptionVisitor.java | 29 +- .../PipeStatementTSStatusVisitor.java | 25 +- .../resource/tsfile/PipeTsFileResourceManager.java | 1 - .../pipe/task/builder/PipeDataNodeTaskBuilder.java | 6 +- .../pipe/task/connection/PipeEventCollector.java | 2 +- .../db/pipe/task/stage/PipeTaskConnectorStage.java | 2 +- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 2 +- .../connector/PipeConnectorSubtaskLifeCycle.java | 2 +- .../connector/PipeConnectorSubtaskManager.java | 4 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 7 + .../iotdb/db/protocol/client/ConfigNodeInfo.java | 9 +- .../impl/DataNodeInternalRPCServiceImpl.java | 2 +- .../execution/exchange/sink/ShuffleSinkHandle.java | 4 +- .../exchange/source/LocalSourceHandle.java | 9 - .../queryengine/execution/memory/MemoryPool.java | 14 +- .../schema/source/TimeSeriesSchemaSource.java | 2 +- .../execution/operator/source/SeriesScanUtil.java | 12 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 11 +- .../analyze/cache/schema/DataNodeSchemaCache.java | 25 + .../analyze/cache/schema/SchemaCacheEntry.java | 8 + .../cache/schema/TimeSeriesSchemaCache.java | 32 + .../cache/schema/dualkeycache/IDualKeyCache.java | 9 + .../schema/dualkeycache/impl/DualKeyCacheImpl.java | 59 ++ .../schema/lastcache/DataNodeLastCacheManager.java | 7 + .../schema/lastcache/ILastCacheContainer.java | 5 +- .../cache/schema/lastcache/LastCacheContainer.java | 7 + .../analyze/schema/AutoCreateSchemaExecutor.java | 6 +- .../config/executor/ClusterConfigTaskExecutor.java | 21 +- .../config/executor/IConfigTaskExecutor.java | 4 +- .../db/queryengine/plan/parser/ASTVisitor.java | 2 +- .../apache/iotdb/db/schemaengine/SchemaEngine.java | 6 +- .../schemaengine/metric/SchemaRegionMemMetric.java | 2 +- .../rescon/DataNodeSchemaQuotaManager.java | 30 +- .../rescon/ISchemaRegionStatistics.java | 2 +- .../rescon/MemSchemaEngineStatistics.java | 29 +- .../rescon/MemSchemaRegionStatistics.java | 40 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 23 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 22 +- .../java/org/apache/iotdb/db/service/DataNode.java | 66 ++ .../db/storageengine/dataregion/DataRegion.java | 44 +- .../exception/CompactionExceptionHandler.java | 15 +- .../execute/recover/CompactionRecoverTask.java | 18 +- .../execute/task/AbstractCompactionTask.java | 15 +- .../execute/task/InnerSpaceCompactionTask.java | 6 +- .../compaction/repair/RepairDataFileScanUtil.java | 4 + .../repair/RepairTimePartitionScanTask.java | 12 + .../schedule/CompactionScheduleTaskManager.java | 4 +- .../compaction/schedule/CompactionScheduler.java | 6 - .../compaction/schedule/CompactionTaskQueue.java | 52 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 6 +- .../memtable/AlignedWritableMemChunk.java | 5 +- .../dataregion/memtable/TsFileProcessor.java | 163 +++-- .../storageengine/dataregion/tsfile/TsFileID.java | 18 +- .../dataregion/tsfile/TsFileManager.java | 2 +- .../tsfile/timeindex/DeviceTimeIndex.java | 64 +- .../dataregion/tsfile/timeindex/FileTimeIndex.java | 22 +- .../dataregion/tsfile/timeindex/ITimeIndex.java | 2 - .../wal/recover/file/TsFilePlanRedoer.java | 8 +- .../file/UnsealedTsFileRecoverPerformer.java | 12 +- .../agent/SubscriptionAgentLauncher.java | 2 +- .../agent/SubscriptionConsumerAgent.java | 2 +- .../broker/SerializedEnrichedEvent.java | 18 +- .../executor/SubscriptionSubtaskExecutor.java | 2 +- .../receiver/SubscriptionReceiverV1.java | 30 +- .../task/stage/SubscriptionTaskConnectorStage.java | 2 +- .../SubscriptionConnectorSubtaskLifeCycle.java | 2 +- .../SubscriptionConnectorSubtaskManager.java | 6 +- .../cache/dualkeycache/DualKeyCacheTest.java | 114 ++++ .../schemaRegion/SchemaStatisticsTest.java | 12 +- .../iotdb/db/pipe/connector/PipeReceiverTest.java | 2 +- .../PipeConnectorSubtaskExecutorTest.java | 1 - .../PipeProcessorSubtaskExecutorTest.java | 1 - .../pattern/CachedSchemaPatternMatcherTest.java | 1 - .../inner/InnerSpaceCompactionExceptionTest.java | 5 +- .../wal/recover/file/TsFilePlanRedoerTest.java | 36 +- .../file/UnsealedTsFileRecoverPerformerTest.java | 87 +++ .../resources/conf/iotdb-common.properties | 9 - .../apache/iotdb/commons/conf/CommonConfig.java | 26 + .../iotdb/commons/conf/CommonDescriptor.java | 5 + .../apache/iotdb/commons/conf/IoTDBConstant.java | 13 +- .../apache/iotdb/commons/path/PathPatternTree.java | 54 ++ .../iotdb/commons/path/PathPatternTreeUtils.java | 22 +- .../iotdb/commons/pipe/config/PipeConfig.java | 7 + .../config/constant/PipeConnectorConstant.java | 18 + .../pipe/connector/PipeReceiverStatusHandler.java | 10 +- .../pipe/connector/client/IoTDBSyncClient.java | 4 +- .../connector/client/IoTDBSyncClientManager.java | 107 ++- .../connector/protocol/IoTDBAirGapConnector.java | 98 ++- .../pipe/connector/protocol/IoTDBConnector.java | 57 +- .../connector/protocol/IoTDBSslSyncConnector.java | 6 +- .../listening/AbstractPipeListeningQueue.java | 6 +- .../iotdb/commons/pipe/event/EnrichedEvent.java | 6 +- .../pipe/metric/PipeEventCommitMetrics.java | 2 +- .../{committer => }/PipeEventCommitManager.java | 2 +- .../{committer => }/PipeEventCommitter.java | 2 +- .../commons/pipe/receiver/IoTDBFileReceiver.java | 263 +++++--- .../commons/pipe/receiver/IoTDBReceiverAgent.java | 46 +- .../meta/consumer/ConsumerGroupMeta.java | 2 +- .../org/apache/iotdb/commons/utils/FileUtils.java | 10 - .../utils/{ => KillPoint}/DataNodeKillPoints.java | 13 +- .../IoTConsensusRemovePeerKillPoints.java} | 13 +- .../iotdb/commons/utils/KillPoint/KillPoint.java | 80 +++ .../NeverTriggeredKillPoint.java} | 10 +- .../consumer/ConsumerGroupDeSerTest.java | 2 +- .../iotdb/tsfile/common/conf/TSFileDescriptor.java | 4 +- .../src/main/thrift/confignode.thrift | 9 +- pom.xml | 3 +- 278 files changed, 5281 insertions(+), 2004 deletions(-)
