This is an automated email from the ASF dual-hosted git repository. spricoder pushed a commit to branch feature/optimize_memory in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8302be7eb1d359ba1e1dfa72b1edc83264335fe1 Merge: a4a0e2948a8 0f797e28cf9 Author: spricoder <[email protected]> AuthorDate: Mon Jul 22 09:41:42 2024 +0800 Merge branch 'master' into feature/optimize_memory .github/workflows/cluster-it-1c1d.yml | 1 + .github/workflows/cluster-it-1c3d.yml | 1 + .github/workflows/daily-it.yml | 1 + .github/workflows/multi-language-client.yml | 20 +- .github/workflows/pipe-it-2cluster.yml | 1 + .github/workflows/sonar-codecov.yml | 1 + .github/workflows/unit-test.yml | 1 + docker/src/main/DockerCompose/do-docker-build.sh | 4 +- docker/src/main/Dockerfile-0.10.1 | 39 -- docker/src/main/Dockerfile-0.11.4 | 41 -- docker/src/main/Dockerfile-0.12.6-cluster | 53 --- docker/src/main/Dockerfile-0.12.6-node | 45 --- docker/src/main/Dockerfile-0.13.0-cluster | 53 --- docker/src/main/Dockerfile-0.13.3-node | 45 --- .../src/main/Dockerfile-0.13.3-node-multi-platform | 45 --- docker/src/main/Dockerfile-0.8.1 | 40 -- docker/src/main/Dockerfile-0.9.1 | 41 -- docker/src/main/Dockerfile-0.9.1-jre8 | 41 -- docker/src/main/Dockerfile-1.0.0-confignode | 2 +- docker/src/main/Dockerfile-1.0.0-datanode | 2 +- docker/src/main/Dockerfile-1.0.0-standalone | 2 +- docker/src/main/Dockerfile-1c1d | 2 +- .../it/env/cluster/config/MppCommonConfig.java | 6 + .../env/cluster/config/MppSharedCommonConfig.java | 7 + .../it/env/remote/config/RemoteCommonConfig.java | 5 + .../org/apache/iotdb/itbase/env/CommonConfig.java | 2 + .../apache/iotdb/db/it/IoTDBDuplicateTimeIT.java | 88 ++++ .../apache/iotdb/db/it/IoTDBInsertWithQueryIT.java | 34 ++ .../db/it/fill/IoTDBFillWithNewDataTypeIT.java | 145 +++++++ .../apache/iotdb/db/it/schema/IoTDBTagAlterIT.java | 1 + .../org/apache/iotdb/db/it/schema/IoTDBTagIT.java | 1 + .../apache/iotdb/db/it/schema/IoTDBTagLimitIT.java | 15 +- .../it/schema/quota/IoTDBClusterDeviceQuotaIT.java | 3 +- .../db/it/schema/quota/IoTDBClusterQuotaIT.java | 1 + .../schema/regionscan/IoTDBActiveRegionScanIT.java | 21 +- .../regionscan/IoTDBActiveRegionScanIT2.java | 15 +- .../IoTDBActiveRegionScanWithModsIT.java | 25 +- .../IoTDBActiveRegionScanWithModsIT2.java | 15 +- .../regionscan/IoTDBActiveRegionScanWithTTLIT.java | 137 +++++++ .../iotdb/db/it/selectinto/IoTDBSelectInto2IT.java | 3 +- .../iotdb/db/it/selectinto/IoTDBSelectInto3IT.java | 3 +- .../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 56 ++- .../iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java | 183 ++++++++- .../pipe/it/autocreate/IoTDBPipeIdempotentIT.java | 84 ++-- .../it/manual/IoTDBPipeMetaLeaderChangeIT.java | 22 +- .../session/it/IoTDBSessionSchemaTemplateIT.java | 4 +- .../it/dual/IoTDBSubscriptionTopicIT.java | 15 +- .../it/local/IoTDBSubscriptionBasicIT.java | 96 ++++- .../org/apache/iotdb/util/AbstractSchemaIT.java | 15 +- .../java/org/apache/iotdb/tool/ExportData.java | 6 +- .../java/org/apache/iotdb/tool/ExportSchema.java | 2 +- .../java/org/apache/iotdb/tool/ImportTsFile.java | 12 +- iotdb-client/client-cpp/pom.xml | 32 +- iotdb-client/client-py/pom.xml | 7 + .../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 + .../rpc/subscription/config/ConsumerConstant.java | 3 +- .../subscription/payload/poll/TabletsPayload.java | 5 +- .../session/subscription/SubscriptionSession.java | 1 + .../consumer/SubscriptionConsumer.java | 29 +- .../consumer/SubscriptionPushConsumer.java | 6 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 8 + .../manager/load/balancer/RegionBalancer.java | 7 +- ...artiteGraphReplicationRegionGroupAllocator.java | 256 ++++++++++++ .../runtime/PipeLeaderChangeHandler.java | 5 +- .../pipe/coordinator/task/PipeTaskCoordinator.java | 22 +- .../coordinator/task/PipeTaskCoordinatorLock.java | 2 +- .../pipe/extractor/ConfigRegionListeningQueue.java | 25 +- .../subscription/SubscriptionCoordinator.java | 18 +- .../persistence/pipe/PipePluginInfo.java | 32 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 73 ++++ .../impl/pipe/AbstractOperatePipeProcedureV2.java | 16 +- .../impl/pipe/plugin/DropPipePluginProcedure.java | 18 +- .../runtime/PipeHandleMetaChangeProcedure.java | 12 + .../impl/pipe/runtime/PipeMetaSyncProcedure.java | 12 + .../impl/pipe/task/AlterPipeProcedureV2.java | 58 ++- .../impl/pipe/task/CreatePipeProcedureV2.java | 4 +- .../impl/schema/DeleteTimeSeriesProcedure.java | 152 +++---- .../AbstractOperateSubscriptionProcedure.java | 16 +- .../runtime/ConsumerGroupMetaSyncProcedure.java | 12 + ...bstractOperateSubscriptionAndPipeProcedure.java | 6 +- .../topic/runtime/TopicMetaSyncProcedure.java | 12 + .../procedure/store/ProcedureFactory.java | 5 +- .../confignode/procedure/store/ProcedureType.java | 1 + .../ConfigRegionListeningQueueTest.java | 13 +- .../impl/pipe/task/AlterPipeProcedureV2Test.java | 10 +- ...reV2Test.java => AlterPipeProcedureV3Test.java} | 19 +- .../org/apache/iotdb/consensus/IStateMachine.java | 9 + .../consensus/iot/IoTConsensusServerImpl.java | 1 + .../consensus/iot/log/ConsensusReqReader.java | 2 +- .../iot/logdispatcher/IndexController.java | 41 +- .../apache/iotdb/consensus/pipe/PipeConsensus.java | 47 ++- .../consensus/pipe/PipeConsensusServerImpl.java | 39 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 16 +- .../dataregion/DataExecutionVisitor.java | 6 + .../dataregion/DataRegionStateMachine.java | 14 +- .../schemaregion/SchemaExecutionVisitor.java | 168 +++++--- .../pipe/agent/plugin/PipeDataNodePluginAgent.java | 35 +- .../db/pipe/agent/runtime/PipeAgentLauncher.java | 8 +- .../client/IoTDBDataNodeAsyncClientManager.java | 71 ++-- .../pipeconsensus/PipeConsensusAsyncConnector.java | 8 +- .../async/IoTDBDataRegionAsyncConnector.java | 6 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 7 + .../dataregion/IoTDBDataRegionExtractor.java | 27 +- .../PipeHistoricalDataRegionTsFileExtractor.java | 6 +- ... PipeRealtimeDataRegionHeartbeatExtractor.java} | 58 +-- .../schemaregion/SchemaRegionListeningQueue.java | 22 +- .../pipeconsensus/PipeConsensusProcessor.java | 7 +- .../pipeconsensus/PipeConsensusReceiver.java | 3 +- .../PipeRealtimePriorityBlockingQueue.java | 54 ++- .../iotdb/db/protocol/client/ConfigNodeClient.java | 4 +- .../impl/DataNodeInternalRPCServiceImpl.java | 50 ++- .../db/queryengine/common/MPPQueryContext.java | 11 + .../execution/executor/RegionWriteExecutor.java | 81 ++-- .../fragment/FragmentInstanceContext.java | 73 ++-- .../fragment/FragmentInstanceExecution.java | 1 + .../execution/fragment/QueryStatistics.java | 229 +++++++++-- .../execution/load/LoadTsFileManager.java | 26 +- .../execution/load/LoadTsFileRateLimiter.java | 2 +- .../execution/operator/ExplainAnalyzeOperator.java | 7 +- .../operator/process/AbstractIntoOperator.java | 12 + .../operator/process/FilterAndProjectOperator.java | 4 +- .../AbstractRegionScanForActiveDataUtil.java | 16 +- .../source/ActiveDeviceRegionScanOperator.java | 4 +- .../source/ActiveTimeSeriesRegionScanOperator.java | 3 +- .../execution/operator/source/FileLoaderUtils.java | 99 +++-- .../source/RegionScanForActiveDeviceUtil.java | 17 +- .../source/RegionScanForActiveTimeSeriesUtil.java | 13 +- .../execution/operator/source/SeriesScanUtil.java | 24 +- .../metric/SeriesScanCostMetricSet.java | 49 +++ .../analyze/cache/partition/PartitionCache.java | 20 +- .../config/executor/ClusterConfigTaskExecutor.java | 6 + .../db/queryengine/plan/parser/ASTVisitor.java | 13 + .../plan/planner/LocalExecutionPlanner.java | 5 +- .../plan/planner/OperatorTreeGenerator.java | 41 +- .../plan/planner/plan/node/PlanNodeType.java | 2 + .../ContinuousSameSearchIndexSeparatorNode.java} | 31 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 72 ++-- .../plan/scheduler/SimpleQueryTerminator.java | 96 ++--- .../plan/statement/literal/BooleanLiteral.java | 4 +- .../plan/statement/literal/DoubleLiteral.java | 3 +- .../plan/statement/literal/Literal.java | 4 + .../plan/statement/literal/LongLiteral.java | 26 +- .../plan/statement/literal/StringLiteral.java | 21 +- .../metadata/pipe/AlterPipeStatement.java | 18 + .../statement/sys/ExplainAnalyzeStatement.java | 6 + .../FragmentInstanceStatisticsDrawer.java | 32 ++ .../statistics/QueryPlanStatistics.java | 9 + .../schemaengine/schemaregion/ISchemaRegion.java | 20 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 146 ++++--- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 155 +++++--- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 120 +++--- .../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 131 ++++-- .../mtree/impl/pbtree/mnode/ICachedMNode.java | 1 + .../schemaregion/mtree/traverser/Traverser.java | 7 + .../write/req/SchemaRegionWritePlanFactory.java | 2 +- .../req/impl/CreateAlignedTimeSeriesPlanImpl.java | 45 ++- .../write/req/impl/CreateTimeSeriesPlanImpl.java | 9 + .../java/org/apache/iotdb/db/service/DataNode.java | 2 +- .../metrics/IoTDBInternalLocalReporter.java | 1 + .../iotdb/db/storageengine/StorageEngine.java | 6 +- .../db/storageengine/dataregion/DataRegion.java | 34 +- .../execute/utils/reader/PointPriorityReader.java | 12 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 75 +++- .../utils/InsertionCompactionCandidateStatus.java} | 17 +- .../dataregion/flush/CompressionRatio.java | 46 ++- .../dataregion/memtable/AbstractMemTable.java | 16 +- .../memtable/AlignedWritableMemChunk.java | 38 +- .../dataregion/memtable/IMemTable.java | 7 +- .../dataregion/memtable/IWritableMemChunk.java | 20 +- .../dataregion/memtable/TsFileProcessor.java | 26 +- .../dataregion/memtable/WritableMemChunk.java | 77 ++-- .../read/reader/chunk/DiskAlignedChunkLoader.java | 4 +- .../read/reader/chunk/DiskChunkLoader.java | 4 +- .../read/reader/chunk/MemAlignedChunkLoader.java | 4 +- .../read/reader/chunk/MemChunkLoader.java | 4 +- .../read/reader/common/PriorityMergeReader.java | 24 +- .../dataregion/snapshot/SnapshotTaker.java | 25 ++ .../dataregion/tsfile/TsFileResource.java | 26 +- .../dataregion/wal/buffer/AbstractWALBuffer.java | 4 +- .../dataregion/wal/buffer/IWALBuffer.java | 4 +- .../dataregion/wal/buffer/WALBuffer.java | 20 +- .../dataregion/wal/buffer/WALEntry.java | 3 + .../dataregion/wal/buffer/WALEntryType.java | 1 + .../dataregion/wal/buffer/WALInfoEntry.java | 1 + .../wal/checkpoint/CheckpointManager.java | 3 +- .../wal/exception/BrokenWALFileException.java} | 17 +- .../storageengine/dataregion/wal/io/LogWriter.java | 9 +- .../dataregion/wal/io/WALMetaData.java | 64 +-- .../storageengine/dataregion/wal/io/WALWriter.java | 2 - .../dataregion/wal/node/IWALNode.java | 4 + .../dataregion/wal/node/WALFakeNode.java | 7 + .../storageengine/dataregion/wal/node/WALNode.java | 17 +- .../rescon/disk/strategy/SequenceStrategy.java | 24 +- .../agent/SubscriptionBrokerAgent.java | 28 +- .../agent/SubscriptionConsumerAgent.java | 12 +- .../subscription/agent/SubscriptionTopicAgent.java | 10 +- .../broker/SubscriptionBlockingPendingQueue.java} | 25 +- .../db/subscription/broker/SubscriptionBroker.java | 95 +++-- .../broker/SubscriptionPrefetchingQueue.java | 72 ++-- .../broker/SubscriptionPrefetchingTabletQueue.java | 21 +- .../broker/SubscriptionPrefetchingTsFileQueue.java | 181 +++++---- .../TsFileDeduplicationBlockingPendingQueue.java | 81 ++++ .../db/subscription/event/SubscriptionEvent.java | 18 +- .../batch/SubscriptionPipeTabletEventBatch.java | 2 +- .../batch/SubscriptionPipeTsFileEventBatch.java | 2 +- .../receiver/SubscriptionReceiverV1.java | 90 +++-- .../org/apache/iotdb/db/utils/CommonUtils.java | 41 +- .../iotdb/db/utils/TimeFilterForDeviceTTL.java | 76 ++++ .../db/utils/datastructure/AlignedTVList.java | 4 +- .../iotdb/db/utils/datastructure/BinaryTVList.java | 4 +- .../iotdb/db/utils/datastructure/TVList.java | 6 +- .../schemaRegion/AbstractSchemaRegionTest.java | 9 +- .../schemaRegion/SchemaRegionAliasAndTagTest.java | 8 +- .../schemaRegion/SchemaRegionBasicTest.java | 382 +++++++++++++----- .../schemaRegion/SchemaRegionManagementTest.java | 4 +- .../SchemaRegionSimpleRecoverTest.java | 135 ++++++- .../schemaRegion/SchemaRegionTemplateTest.java | 15 +- .../schemaRegion/SchemaRegionTestUtil.java | 52 ++- .../schemaRegion/SchemaStatisticsTest.java | 146 +++---- .../agent/plugin/PipeDataNodePluginAgentTest.java | 39 +- .../extractor/SchemaRegionListeningQueueTest.java | 15 +- .../compaction/CompactionTaskManagerTest.java | 5 +- .../InsertionCrossSpaceCompactionSelectorTest.java | 441 +-------------------- .../dataregion/flush/CompressionRatioTest.java | 36 ++ .../dataregion/wal/io/WALFileTest.java | 15 + .../org/apache/iotdb/db/tools/MLogParserTest.java | 2 +- .../apache/iotdb/db/utils/DateTimeUtilsTest.java | 63 +-- .../db/utils/SchemaRegionSnapshotParserTest.java | 10 +- .../reporter/iotdb/IoTDBSessionReporter.java | 2 +- .../conf/iotdb-system.properties.template | 7 +- .../apache/iotdb/commons/conf/CommonConfig.java | 32 +- .../iotdb/commons/conf/CommonDescriptor.java | 17 +- .../commons/executable/ExecutableManager.java | 3 + .../pipe/agent/plugin/PipePluginConstructor.java | 14 +- .../iotdb/commons/pipe/config/PipeConfig.java | 16 +- .../pipe/connector/limiter/GlobalRateLimiter.java | 2 +- .../connector/limiter/PipeEndPointRateLimiter.java | 2 +- .../pipe/plugin/meta/PipePluginMetaKeeper.java | 35 +- .../service/PipePluginClassLoaderManager.java | 42 +- .../service/PipePluginExecutableManager.java | 43 +- .../connection/UnboundedBlockingPendingQueue.java | 2 +- .../schema/node/role/IMeasurementMNode.java | 2 +- .../commons/schema/node/utils/IMNodeFactory.java | 1 + .../subscription/config/SubscriptionConfig.java | 7 + .../org/apache/iotdb/commons/utils/IOUtils.java | 32 ++ .../apache/iotdb/commons/utils/JVMCommonUtils.java | 30 +- .../pipe/plugin/meta/PipePluginMetaTest.java | 2 +- .../src/main/thrift/confignode.thrift | 2 + .../src/main/thrift/datanode.thrift | 9 +- pom.xml | 14 +- 250 files changed, 5160 insertions(+), 2775 deletions(-)
