This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch beyyes/optimize_for_multiply_device_limit_query in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5e21507161383494da560be04a9f093056ed4ad2 Merge: 88d24077b3a 98310822b32 Author: Beyyes <[email protected]> AuthorDate: Sun Jan 28 12:21:04 2024 +0800 Merge branch 'master' of github.com:apache/iotdb .../it/cluster/IoTDBClusterRestartIT.java | 70 ++ .../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 9 + .../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 132 ++ .../IoTDBOrderByLimitOffsetAlignByDeviceIT.java | 1 + .../src/assembly/resources/tools/collect-info.bat | 146 ++- .../org/apache/iotdb/isession/SessionConfig.java | 4 + .../java/org/apache/iotdb/session/Session.java | 27 +- .../apache/iotdb/session/SessionConnection.java | 660 ++++++++-- .../org/apache/iotdb/session/pool/SessionPool.java | 1289 +++++++++----------- .../iotdb/session/SessionConnectionTest.java | 9 +- .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 + .../src/assembly/resources/conf/confignode-env.bat | 2 +- .../resources/conf/iotdb-confignode.properties | 5 - .../assembly/resources/sbin/daemon-confignode.sh | 22 +- .../confignode/client/DataNodeRequestType.java | 1 + .../client/async/AsyncDataNodeClientPool.java | 5 + .../client/async/handlers/AsyncClientHandler.java | 1 + .../heartbeat/DataNodeHeartbeatHandler.java | 2 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 17 +- .../confignode/conf/ConfigNodeDescriptor.java | 8 - .../iotdb/confignode/manager/ConfigManager.java | 16 +- .../apache/iotdb/confignode/manager/IManager.java | 3 + .../manager/consensus/ConsensusManager.java | 2 - .../confignode/manager/load/cache/LoadCache.java | 2 +- .../manager/load/cache/node/BaseNodeCache.java | 2 +- .../load/cache/node/ConfigNodeHeartbeatCache.java | 2 +- .../load/cache/node/DataNodeHeartbeatCache.java | 2 +- .../load/cache/node/NodeHeartbeatSample.java | 6 +- .../manager/load/cache/node/NodeStatistics.java | 2 +- .../manager/load/cache/region/RegionCache.java | 5 +- .../load/cache/region/RegionHeartbeatSample.java | 2 +- .../manager/load/service/HeartbeatService.java | 4 +- .../iotdb/confignode/manager/node/NodeManager.java | 9 + .../persistence/executor/ConfigPlanExecutor.java | 9 +- .../partition/DatabasePartitionTable.java | 11 + .../persistence/partition/PartitionInfo.java | 19 + .../persistence/partition/RegionGroup.java | 15 + .../procedure/env/ConfigNodeProcedureEnv.java | 2 +- .../iotdb/confignode/service/ConfigNode.java | 12 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 7 +- .../router/priority/GreedyPriorityTest.java | 6 +- .../priority/LeaderPriorityBalancerTest.java | 6 +- .../manager/load/cache/LoadCacheTest.java | 2 +- .../manager/load/cache/NodeCacheTest.java | 6 +- .../manager/load/cache/RegionGroupCacheTest.java | 6 +- .../manager/load/cache/RegionRouteCacheTest.java | 2 +- .../iotdb/consensus/config/IoTConsensusConfig.java | 16 - .../apache/iotdb/consensus/config/RatisConfig.java | 30 - .../exception/RatisReadUnavailableException.java | 13 +- .../iot/client/IoTConsensusClientPool.java | 2 - .../consensus/iot/logdispatcher/LogDispatcher.java | 24 +- .../iotdb/consensus/ratis/RatisConsensus.java | 20 +- .../apache/iotdb/consensus/ratis/utils/Utils.java | 6 +- .../iotdb/consensus/ratis/RatisConsensusTest.java | 4 +- .../iotdb/consensus/ratis/RecoverReadTest.java | 6 +- .../src/assembly/resources/conf/datanode-env.bat | 2 +- .../resources/conf/iotdb-datanode.properties | 7 +- .../src/assembly/resources/sbin/daemon-datanode.sh | 22 +- .../org/apache/iotdb/db/audit/AuditLogger.java | 1 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 53 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 44 +- .../db/consensus/DataRegionConsensusImpl.java | 2 - .../db/consensus/SchemaRegionConsensusImpl.java | 1 - .../IoTConsensusDataRegionStateMachine.java | 2 +- .../schemaregion/SchemaExecutionVisitor.java | 1 - .../agent/runtime/PipePeriodicalJobExecutor.java | 1 + .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 5 + .../db/pipe/agent/task/PipeTaskDataNodeAgent.java | 77 ++ .../apache/iotdb/db/pipe/event/EnrichedEvent.java | 41 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 13 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 11 + .../common/tsfile/PipeTsFileInsertionEvent.java | 19 +- .../pipe/extractor/IoTDBDataRegionExtractor.java | 34 +- .../PipeHistoricalDataRegionTsFileExtractor.java | 64 +- .../PipeRealtimeDataRegionHybridExtractor.java | 9 +- .../iotdb/db/pipe/metric/PipeExtractorMetrics.java | 7 +- .../db/pipe/task/subtask/PipeDataNodeSubtask.java | 105 +- .../subtask/connector/PipeConnectorSubtask.java | 170 +-- .../connector/PipeConnectorSubtaskLifeCycle.java | 42 +- .../subtask/processor/PipeProcessorSubtask.java | 14 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 6 + .../protocol/client/DataNodeClientPoolFactory.java | 2 - .../impl/DataNodeInternalRPCServiceImpl.java | 24 + .../db/queryengine/common/MPPQueryContext.java | 10 +- .../common/header/ColumnHeaderConstant.java | 1 + .../execution/load/LoadTsFileManager.java | 144 ++- .../execution/operator/AggregationUtil.java | 6 + .../execution/operator/process/TopKOperator.java | 42 +- .../AbstractSeriesAggregationScanOperator.java | 2 +- .../operator/source/AlignedSeriesScanOperator.java | 4 +- .../operator/source/SeriesScanOperator.java | 3 +- .../execution/schedule/DriverScheduler.java | 2 +- .../queryengine/load/LoadTsFileMemoryManager.java | 5 +- .../iotdb/db/queryengine/plan/Coordinator.java | 16 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 9 +- .../plan/analyze/LoadTsfileAnalyzer.java | 9 +- .../plan/analyze/schema/ClusterSchemaFetcher.java | 6 +- .../plan/execution/config/ConfigTaskVisitor.java | 8 + .../config/executor/ClusterConfigTaskExecutor.java | 49 + .../config/executor/IConfigTaskExecutor.java | 2 + .../execution/config/metadata/ShowRegionTask.java | 7 +- .../plan/execution/config/sys/RepairDataTask.java | 42 + .../db/queryengine/plan/parser/ASTVisitor.java | 12 + .../plan/planner/LocalExecutionPlanner.java | 4 +- .../plan/planner/LogicalPlanBuilder.java | 25 +- .../plan/planner/OperatorTreeGenerator.java | 4 +- .../plan/planner/distribution/SourceRewriter.java | 9 +- .../plan/parameter/AggregationDescriptor.java | 14 +- .../plan/scheduler/AsyncSendPlanNodeHandler.java | 4 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 52 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 123 +- .../queryengine/plan/statement/StatementType.java | 1 + .../plan/statement/StatementVisitor.java | 5 + .../plan/statement/sys/RepairDataStatement.java | 62 + .../schemaregion/impl/SchemaRegionMemoryImpl.java | 1 + .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 1 + .../mtree/impl/pbtree/CachedMTreeStore.java | 2 +- .../schemafile/pagemgr/BTreePageManager.java | 8 +- .../pbtree/schemafile/pagemgr/PageIOChannel.java | 184 +++ .../schemafile/pagemgr/PageIndexSortBuckets.java | 129 ++ .../pbtree/schemafile/pagemgr/PageManager.java | 452 +------ .../impl/pbtree/schemafile/pagemgr/PagePool.java | 152 +++ .../schemafile/pagemgr/SchemaPageContext.java | 109 ++ .../java/org/apache/iotdb/db/service/DataNode.java | 24 +- .../iotdb/db/storageengine/StorageEngine.java | 64 +- .../db/storageengine/dataregion/DataRegion.java | 33 +- ...paceEstimator.java => DeviceLastFlushTime.java} | 40 +- .../dataregion/HashLastFlushTimeMap.java | 92 +- .../storageengine/dataregion/ILastFlushTime.java} | 18 +- .../dataregion/ILastFlushTimeMap.java | 12 +- .../dataregion/PartitionLastFlushTime.java} | 31 +- .../impl/ReadChunkCompactionPerformer.java | 19 +- .../impl/ReadPointCompactionPerformer.java | 14 +- .../RepairUnsortedFileCompactionPerformer.java | 86 ++ .../execute/task/AbstractCompactionTask.java | 9 +- .../execute/task/CompactionTaskPriorityType.java | 14 +- .../execute/task/CompactionTaskSummary.java | 44 +- .../execute/task/CrossSpaceCompactionTask.java | 5 +- .../execute/task/InnerSpaceCompactionTask.java | 35 +- .../task/InsertionCrossSpaceCompactionTask.java | 15 +- .../task/RepairUnsortedFileCompactionTask.java | 239 ++++ .../execute/utils/MultiTsFileDeviceIterator.java | 4 + .../executor/ModifiedStatus.java} | 16 +- .../fast/AlignedSeriesCompactionExecutor.java | 1 + .../fast/NonAlignedSeriesCompactionExecutor.java | 1 + .../executor/fast/SeriesCompactionExecutor.java | 6 +- .../ReadChunkAlignedSeriesCompactionExecutor.java | 465 +++++++ .../readchunk/SingleSeriesCompactionExecutor.java | 4 +- .../executor/readchunk/loader/ChunkLoader.java | 104 ++ .../readchunk/loader/InstantChunkLoader.java | 105 ++ .../readchunk/loader/InstantPageLoader.java | 103 ++ .../executor/readchunk/loader/PageLoader.java | 107 ++ .../execute/utils/reader/PointPriorityReader.java | 2 +- .../utils/writer/AbstractCompactionWriter.java | 6 +- .../writer/AbstractCrossCompactionWriter.java | 6 +- .../writer/AbstractInnerCompactionWriter.java | 6 +- .../utils/writer/FastCrossCompactionWriter.java | 6 +- .../utils/writer/FastInnerCompactionWriter.java | 6 +- .../writer/ReadPointCrossCompactionWriter.java | 7 +- .../writer/ReadPointInnerCompactionWriter.java | 7 +- .../writer/RepairUnsortedFileCompactionWriter.java | 109 ++ .../compaction/io/CompactionTsFileReader.java | 18 + .../dataregion/compaction/repair/RepairLogger.java | 136 +++ .../repair/RepairTaskRecoverLogParser.java | 115 ++ .../compaction/repair/RepairTimePartition.java | 136 +++ .../repair/UnsortedFileRepairTaskScheduler.java | 350 ++++++ .../compaction/schedule/CompactionScheduler.java | 24 +- .../compaction/schedule/CompactionTaskManager.java | 4 +- .../DefaultCompactionTaskComparatorImpl.java | 10 +- .../estimator/AbstractCompactionEstimator.java | 10 + .../estimator/AbstractCrossSpaceEstimator.java | 3 + .../estimator/AbstractInnerSpaceEstimator.java | 19 +- .../RepairUnsortedFileCompactionEstimator.java | 72 ++ .../impl/SizeTieredCompactionSelector.java | 81 +- .../InsertionCrossCompactionTaskResource.java | 25 + .../selector/utils/TsFileResourceCandidate.java | 5 +- .../compaction/settle/SettleRequestHandler.java | 4 +- .../dataregion/memtable/AbstractMemTable.java | 18 + .../dataregion/memtable/IMemTable.java | 2 + .../dataregion/memtable/TsFileProcessor.java | 7 + .../dataregion/tsfile/TsFileManager.java | 75 +- .../TsFileRepairStatus.java} | 14 +- .../dataregion/tsfile/TsFileResource.java | 16 + .../tsfile/generator/TsFileNameGenerator.java | 53 +- .../dataregion/wal/buffer/WALBuffer.java | 57 +- .../wal/checkpoint/CheckpointManager.java | 50 +- .../dataregion/wal/checkpoint/MemTableInfo.java | 22 +- .../dataregion/wal/io/WALByteBufReader.java | 26 +- .../dataregion/wal/io/WALMetaData.java | 76 +- .../storageengine/dataregion/wal/node/WALNode.java | 216 ++-- .../dataregion/wal/recover/WALNodeRecoverTask.java | 41 +- .../wal/recover/file/TsFilePlanRedoer.java | 2 + .../rescon/memory/TimePartitionInfo.java | 4 + .../rescon/memory/TimePartitionManager.java | 25 +- .../TsFileOverlapValidationAndRepairTool.java | 6 - .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 32 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 27 +- .../SchemaRegionSimpleRecoverTest.java | 98 ++ .../execution/operator/OperatorMemoryTest.java | 19 +- .../plan/planner/PipelineBuilderTest.java | 55 + .../AlignByTimeOrderByLimitOffsetTest.java | 5 +- .../storageengine/dataregion/DataRegionTest.java | 10 +- .../dataregion/LastFlushTimeMapTest.java | 84 +- .../FastInnerCompactionPerformerTest.java | 108 ++ .../compaction/ReadChunkInnerCompactionTest.java | 144 +++ ...eCompactionWithFastPerformerValidationTest.java | 4 +- ...actionWithReadPointPerformerValidationTest.java | 2 +- .../InsertionCrossSpaceCompactionSelectorTest.java | 4 +- .../cross/InsertionCrossSpaceCompactionTest.java | 6 +- .../compaction/inner/InnerCompactionTest.java | 1 + ...nkCompactionPerformerWithAlignedSeriesTest.java | 768 ++++++++++++ .../compaction/repair/RepairLoggerTest.java | 118 ++ .../repair/RepairUnsortedFileCompactionTest.java | 973 +++++++++++++++ .../compaction/utils/CompactionCheckerUtils.java | 8 +- .../compaction/utils/CompactionTestFileWriter.java | 8 +- .../utils/MultiTsFileDeviceIteratorTest.java | 12 +- .../dataregion/wal/node/WALEntryHandlerTest.java | 13 +- .../wal/node/WalDeleteOutdatedNewTest.java | 585 +++++++++ .../wal/recover/WALRecoverWriterTest.java | 9 +- .../rescon/memory/TimePartitionManagerTest.java | 18 + .../resources/conf/iotdb-cluster.properties | 5 +- .../resources/conf/iotdb-common.properties | 45 +- .../src/assembly/resources/sbin/destroy-all.sh | 1 + .../iotdb/commons/auth/role/BasicRoleManager.java | 1 + .../iotdb/commons/client/ClientPoolFactory.java | 29 +- .../client/property/ClientPoolProperty.java | 35 +- .../apache/iotdb/commons/conf/CommonConfig.java | 52 +- .../iotdb/commons/conf/CommonDescriptor.java | 33 +- .../commons/pipe/agent/task/PipeTaskAgent.java | 4 +- .../iotdb/commons/pipe/config/PipeConfig.java | 20 +- .../apache/iotdb/commons/utils/StatusUtils.java | 52 + .../iotdb/commons/client/ClientManagerTest.java | 47 +- .../file/metadata/AlignedTimeSeriesMetadata.java | 1 + .../iotdb/tsfile/read/TsFileSequenceReader.java | 3 +- .../tsfile/read/reader/page/AlignedPageReader.java | 7 +- .../thrift-commons/src/main/thrift/common.thrift | 1 + .../src/main/thrift/confignode.thrift | 4 + .../src/main/thrift/datanode.thrift | 2 + pom.xml | 2 +- 241 files changed, 9474 insertions(+), 2770 deletions(-)
