This is an automated email from the ASF dual-hosted git repository. chenyz pushed a commit to branch pbtree_page_concurrency in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0c64aea41a353e1f5816737b2180b34b3ba6b398 Merge: c9148d155a6 82d04599f6c Author: Chen YZ <[email protected]> AuthorDate: Thu Jan 25 13:22:56 2024 +0800 Merge branch 'master' into pbtree_page_concurrency .../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 9 + .../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 130 +++ .../src/assembly/resources/tools/collect-info.bat | 146 ++-- .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 + .../resources/conf/iotdb-confignode.properties | 5 - .../assembly/resources/sbin/daemon-confignode.sh | 18 +- .../confignode/client/DataNodeRequestType.java | 1 + .../client/async/AsyncDataNodeClientPool.java | 5 + .../client/async/handlers/AsyncClientHandler.java | 1 + .../heartbeat/DataNodeHeartbeatHandler.java | 2 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 17 +- .../confignode/conf/ConfigNodeDescriptor.java | 8 - .../statemachine/ConfigRegionStateMachine.java | 23 + .../iotdb/confignode/manager/ConfigManager.java | 16 +- .../apache/iotdb/confignode/manager/IManager.java | 3 + .../manager/consensus/ConsensusManager.java | 2 - .../confignode/manager/load/cache/LoadCache.java | 2 +- .../manager/load/cache/node/BaseNodeCache.java | 2 +- .../load/cache/node/ConfigNodeHeartbeatCache.java | 2 +- .../load/cache/node/DataNodeHeartbeatCache.java | 2 +- .../load/cache/node/NodeHeartbeatSample.java | 6 +- .../manager/load/cache/node/NodeStatistics.java | 2 +- .../manager/load/cache/region/RegionCache.java | 5 +- .../load/cache/region/RegionHeartbeatSample.java | 2 +- .../manager/load/service/HeartbeatService.java | 4 +- .../iotdb/confignode/manager/node/NodeManager.java | 9 + .../procedure/env/ConfigNodeProcedureEnv.java | 2 +- .../iotdb/confignode/service/ConfigNode.java | 12 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 7 +- .../router/priority/GreedyPriorityTest.java | 6 +- .../priority/LeaderPriorityBalancerTest.java | 6 +- .../manager/load/cache/LoadCacheTest.java | 2 +- .../manager/load/cache/NodeCacheTest.java | 6 +- .../manager/load/cache/RegionGroupCacheTest.java | 6 +- .../manager/load/cache/RegionRouteCacheTest.java | 2 +- .../iotdb/consensus/config/IoTConsensusConfig.java | 16 - .../apache/iotdb/consensus/config/RatisConfig.java | 16 - .../exception/RatisReadUnavailableException.java | 13 +- .../iot/client/IoTConsensusClientPool.java | 2 - .../consensus/iot/logdispatcher/LogDispatcher.java | 24 +- .../iotdb/consensus/ratis/RatisConsensus.java | 18 +- .../resources/conf/iotdb-datanode.properties | 5 - .../src/assembly/resources/sbin/daemon-datanode.sh | 18 +- .../org/apache/iotdb/db/audit/AuditLogger.java | 1 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 42 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 27 +- .../db/consensus/DataRegionConsensusImpl.java | 2 - .../db/consensus/SchemaRegionConsensusImpl.java | 1 - .../agent/runtime/PipePeriodicalJobExecutor.java | 1 + .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 5 + .../db/pipe/agent/task/PipeTaskDataNodeAgent.java | 77 ++ .../pipe/extractor/IoTDBDataRegionExtractor.java | 11 + .../PipeRealtimeDataRegionHybridExtractor.java | 9 +- .../iotdb/db/pipe/metric/PipeExtractorMetrics.java | 7 +- .../db/pipe/task/subtask/PipeDataNodeSubtask.java | 105 ++- .../subtask/connector/PipeConnectorSubtask.java | 150 ++-- .../iotdb/db/protocol/client/ConfigNodeClient.java | 6 + .../protocol/client/DataNodeClientPoolFactory.java | 2 - .../impl/DataNodeInternalRPCServiceImpl.java | 24 + .../db/queryengine/common/MPPQueryContext.java | 10 +- .../execution/load/LoadTsFileManager.java | 7 +- .../execution/operator/AggregationUtil.java | 6 + .../queryengine/load/LoadTsFileMemoryManager.java | 5 +- .../iotdb/db/queryengine/plan/Coordinator.java | 5 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 9 +- .../plan/analyze/LoadTsfileAnalyzer.java | 9 +- .../plan/analyze/schema/ClusterSchemaFetcher.java | 6 +- .../plan/execution/config/ConfigTaskVisitor.java | 8 + .../config/executor/ClusterConfigTaskExecutor.java | 49 ++ .../config/executor/IConfigTaskExecutor.java | 2 + .../plan/execution/config/sys/RepairDataTask.java | 42 + .../db/queryengine/plan/parser/ASTVisitor.java | 12 + .../plan/planner/LogicalPlanBuilder.java | 19 +- .../plan/planner/distribution/SourceRewriter.java | 9 +- .../plan/parameter/AggregationDescriptor.java | 14 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 16 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 123 +-- .../queryengine/plan/statement/StatementType.java | 1 + .../plan/statement/StatementVisitor.java | 5 + .../plan/statement/sys/RepairDataStatement.java | 62 ++ .../schemafile/pagemgr/BTreePageManager.java | 8 +- .../pbtree/schemafile/pagemgr/PageIOChannel.java | 184 ++++ .../schemafile/pagemgr/PageIndexSortBuckets.java | 129 +++ .../pbtree/schemafile/pagemgr/PageManager.java | 452 +--------- .../impl/pbtree/schemafile/pagemgr/PagePool.java | 152 ++++ .../schemafile/pagemgr/SchemaPageContext.java | 109 +++ .../java/org/apache/iotdb/db/service/DataNode.java | 24 +- .../iotdb/db/storageengine/StorageEngine.java | 45 + .../db/storageengine/dataregion/DataRegion.java | 16 +- .../impl/ReadChunkCompactionPerformer.java | 7 +- .../impl/ReadPointCompactionPerformer.java | 14 +- .../RepairUnsortedFileCompactionPerformer.java | 86 ++ .../execute/task/AbstractCompactionTask.java | 9 +- .../execute/task/CompactionTaskPriorityType.java | 14 +- .../execute/task/CrossSpaceCompactionTask.java | 5 +- .../execute/task/InnerSpaceCompactionTask.java | 35 +- .../task/InsertionCrossSpaceCompactionTask.java | 12 +- .../task/RepairUnsortedFileCompactionTask.java | 239 +++++ .../ReadChunkAlignedSeriesCompactionExecutor.java | 11 +- .../execute/utils/reader/PointPriorityReader.java | 2 +- .../utils/writer/AbstractCompactionWriter.java | 6 +- .../writer/AbstractCrossCompactionWriter.java | 6 +- .../writer/AbstractInnerCompactionWriter.java | 6 +- .../utils/writer/FastCrossCompactionWriter.java | 6 +- .../utils/writer/FastInnerCompactionWriter.java | 6 +- .../writer/ReadPointCrossCompactionWriter.java | 7 +- .../writer/ReadPointInnerCompactionWriter.java | 7 +- .../writer/RepairUnsortedFileCompactionWriter.java | 109 +++ .../dataregion/compaction/repair/RepairLogger.java | 136 +++ .../repair/RepairTaskRecoverLogParser.java | 115 +++ .../compaction/repair/RepairTimePartition.java | 136 +++ .../repair/UnsortedFileRepairTaskScheduler.java | 350 ++++++++ .../compaction/schedule/CompactionScheduler.java | 22 +- .../compaction/schedule/CompactionTaskManager.java | 1 - .../DefaultCompactionTaskComparatorImpl.java | 10 +- .../RepairUnsortedFileCompactionEstimator.java | 72 ++ .../impl/SizeTieredCompactionSelector.java | 40 +- .../InsertionCrossCompactionTaskResource.java | 25 + .../selector/utils/TsFileResourceCandidate.java | 5 +- .../compaction/settle/SettleRequestHandler.java | 4 +- .../dataregion/memtable/AbstractMemTable.java | 18 + .../dataregion/memtable/IMemTable.java | 2 + .../dataregion/memtable/TsFileProcessor.java | 5 + .../dataregion/tsfile/TsFileManager.java | 39 +- .../TsFileRepairStatus.java} | 14 +- .../dataregion/tsfile/TsFileResource.java | 15 + .../tsfile/generator/TsFileNameGenerator.java | 53 +- .../dataregion/wal/buffer/WALBuffer.java | 57 +- .../wal/checkpoint/CheckpointManager.java | 50 +- .../dataregion/wal/checkpoint/MemTableInfo.java | 22 +- .../dataregion/wal/io/WALByteBufReader.java | 26 +- .../dataregion/wal/io/WALMetaData.java | 76 +- .../storageengine/dataregion/wal/node/WALNode.java | 216 ++--- .../dataregion/wal/recover/WALNodeRecoverTask.java | 41 +- .../TsFileOverlapValidationAndRepairTool.java | 6 - .../org/apache/iotdb/db/utils/SchemaUtils.java | 27 +- .../storageengine/dataregion/DataRegionTest.java | 4 + .../FastInnerCompactionPerformerTest.java | 108 +++ ...eCompactionWithFastPerformerValidationTest.java | 4 +- ...actionWithReadPointPerformerValidationTest.java | 2 +- .../compaction/inner/InnerCompactionTest.java | 1 + ...nkCompactionPerformerWithAlignedSeriesTest.java | 155 +++- .../compaction/repair/RepairLoggerTest.java | 118 +++ .../repair/RepairUnsortedFileCompactionTest.java | 973 +++++++++++++++++++++ .../utils/MultiTsFileDeviceIteratorTest.java | 12 +- .../dataregion/wal/node/WALEntryHandlerTest.java | 13 +- .../wal/node/WalDeleteOutdatedNewTest.java | 585 +++++++++++++ .../wal/recover/WALRecoverWriterTest.java | 9 +- .../resources/conf/iotdb-cluster.properties | 2 +- .../resources/conf/iotdb-common.properties | 35 +- .../iotdb/commons/client/ClientPoolFactory.java | 29 +- .../client/property/ClientPoolProperty.java | 35 +- .../apache/iotdb/commons/conf/CommonConfig.java | 48 +- .../iotdb/commons/conf/CommonDescriptor.java | 33 +- .../commons/pipe/agent/task/PipeTaskAgent.java | 4 +- .../iotdb/commons/pipe/config/PipeConfig.java | 20 +- .../iotdb/commons/client/ClientManagerTest.java | 47 +- .../src/main/thrift/confignode.thrift | 3 + .../src/main/thrift/datanode.thrift | 2 + 161 files changed, 5595 insertions(+), 1420 deletions(-)
