This is an automated email from the ASF dual-hosted git repository. spricoder pushed a commit to branch feature/memory_transfer in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7e61a737c12e804d254be8db59613febd55dab5d Merge: f28f9f719b3 5205e3c9130 Author: spricoder <[email protected]> AuthorDate: Mon Mar 3 14:11:58 2025 +0800 Merge branch 'feature/memory_collect' into feature/memory_transfer dependencies.json | 1 + .../server/CustomizedJsonPayloadFormatter.java | 8 +- .../org/apache/iotdb/mqtt/server/MyMessage.java | 80 --- .../java/org/apache/iotdb/mqtt/MQTTClient.java | 2 +- .../relational/it/db/it/IoTDBInsertTableIT.java | 11 +- .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java | 12 +- .../relational/it/mqtt/IoTDBMQTTServiceIT.java | 2 +- .../IoTDBCorrelatedExistsSubqueryIT.java | 428 +++++++++++++ .../IoTDBUncorrelatedExistsSubqueryIT.java | 391 ++++++++++++ .../it/session/IoTDBSessionRelationalIT.java | 34 - .../iotdb/session/it/IoTDBSessionSimpleIT.java | 48 -- .../org/apache/iotdb/tool/common/OptionsUtil.java | 54 +- .../org/apache/iotdb/tool/data/ExportData.java | 4 +- .../java/org/apache/iotdb/session/Session.java | 17 +- .../SubscriptionTablePushConsumerBuilder.java | 3 +- iotdb-core/confignode/pom.xml | 4 + .../iotdb/confignode/conf/ConfigNodeConfig.java | 45 ++ .../confignode/conf/ConfigNodeDescriptor.java | 30 + .../manager/load/balancer/RouteBalancer.java | 24 +- .../manager/load/cache/AbstractLoadCache.java | 26 +- .../manager/load/cache/IFailureDetector.java} | 24 +- .../manager/load/cache/detector/FixedDetector.java | 58 ++ .../load/cache/detector/PhiAccrualDetector.java | 181 ++++++ .../load/cache/node/AINodeHeartbeatCache.java | 15 +- .../load/cache/node/ConfigNodeHeartbeatCache.java | 12 +- .../load/cache/node/DataNodeHeartbeatCache.java | 11 +- .../manager/load/cache/region/RegionCache.java | 12 +- .../pipe/agent/task/PipeConfigNodeSubtask.java | 2 +- .../pipe/agent/task/PipeConfigNodeTaskAgent.java | 4 +- .../pipe/extractor/IoTDBConfigRegionExtractor.java | 4 +- .../manager/pipe/metric/PipeConfigNodeMetrics.java | 9 + .../PipeConfigNodeRemainingTimeMetrics.java | 2 +- .../PipeConfigNodeRemainingTimeOperator.java | 2 +- .../PipeConfigNodeResourceMetrics.java | 2 +- .../{ => overview}/PipeProcedureMetrics.java | 2 +- .../metric/{ => overview}/PipeTaskInfoMetrics.java | 2 +- .../PipeTemporaryMetaInCoordinatorMetrics.java | 2 +- .../PipeConfigNodeReceiverMetrics.java | 2 +- .../PipeConfigRegionConnectorMetrics.java | 2 +- .../PipeConfigNodeListenerMetrics.java | 2 +- .../PipeConfigRegionExtractorMetrics.java | 2 +- .../receiver/protocol/IoTDBConfigNodeReceiver.java | 2 +- .../resource/PipeConfigNodeResourceManager.java | 1 + .../PipeConfigNodeSnapshotResourceManager.java | 2 +- .../manager/schema/ClusterSchemaManager.java | 12 +- .../iotdb/confignode/persistence/TTLInfo.java | 2 +- .../confignode/persistence/pipe/PipeInfo.java | 2 +- .../persistence/schema/ClusterSchemaInfo.java | 20 + .../procedure/PartitionTableAutoCleaner.java | 14 +- .../impl/pipe/AbstractOperatePipeProcedureV2.java | 2 +- .../impl/pipe/task/AlterPipeProcedureV2.java | 1 + .../manager/load/cache/detector/DetectorTest.java | 179 ++++++ .../common/request/IConsensusRequest.java | 5 + .../common/request/IndexedConsensusRequest.java | 8 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 12 +- iotdb-core/datanode/pom.xml | 8 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- .../PipeDataRegionConnectorConstructor.java | 4 + .../runtime/PipeSchemaRegionListenerManager.java | 2 +- .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 4 +- .../pipe/agent/task/connection/EnrichedDeque.java | 2 +- .../agent/task/connection/PipeEventCollector.java | 14 +- .../agent/task/stage/PipeTaskProcessorStage.java | 5 +- .../subtask/connector/PipeConnectorSubtask.java | 4 +- .../connector/PipeConnectorSubtaskManager.java | 2 +- .../PipeRealtimePriorityBlockingQueue.java | 2 +- .../subtask/processor/PipeProcessorSubtask.java | 4 +- .../connector/protocol/opcda/OpcDaConnector.java | 138 ++++ .../pipe/connector/protocol/opcda/OpcDaHeader.java | 200 ++++++ .../protocol/opcda/OpcDaServerHandle.java | 390 ++++++++++++ .../connector/protocol/opcua/OpcUaConnector.java | 50 +- .../connector/protocol/opcua/OpcUaNameSpace.java | 2 +- .../event/common/heartbeat/PipeHeartbeatEvent.java | 4 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 67 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 30 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 2 +- .../scan/TsFileInsertionEventScanParser.java | 3 - .../table/TsFileInsertionEventTableParser.java | 95 ++- ...ileInsertionEventTableParserTabletIterator.java | 421 ++++++++++--- .../dataregion/IoTDBDataRegionExtractor.java | 4 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 2 +- .../PipeRealtimeDataRegionHybridExtractor.java | 2 +- .../realtime/assigner/PipeDataRegionAssigner.java | 4 +- .../dataregion/realtime/epoch/TsFileEpoch.java | 2 +- .../schemaregion/IoTDBSchemaRegionExtractor.java | 4 +- .../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 12 + .../PipeDataNodeRemainingEventAndTimeMetrics.java | 2 +- .../PipeDataNodeRemainingEventAndTimeOperator.java | 2 +- .../{ => overview}/PipeHeartbeatEventMetrics.java | 2 +- .../metric/{ => overview}/PipeResourceMetrics.java | 2 +- .../PipeWALInsertNodeCacheMetrics.java | 2 +- .../{ => processor}/PipeProcessorMetrics.java | 2 +- .../PipeDataNodeReceiverMetrics.java | 2 +- .../PipeSchemaRegionConnectorMetrics.java | 2 +- .../PipeSchemaRegionExtractorMetrics.java | 2 +- .../PipeSchemaRegionListenerMetrics.java | 2 +- .../{ => sink}/PipeDataRegionConnectorMetrics.java | 2 +- .../metric/{ => source}/PipeAssignerMetrics.java | 2 +- .../{ => source}/PipeDataRegionEventCounter.java | 2 +- .../PipeDataRegionExtractorMetrics.java | 2 +- .../protocol/thrift/IoTDBDataNodeReceiver.java | 2 +- .../resource/memory/InsertNodeMemoryEstimator.java | 2 +- .../pipe/resource/memory/PipeMemoryWeightUtil.java | 65 ++ .../db/protocol/mqtt/JSONPayloadFormatter.java | 7 +- .../db/protocol/mqtt/LinePayloadFormatter.java | 7 +- .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 11 +- .../iotdb/db/protocol/mqtt/PayloadFormatter.java | 6 + .../protocol/thrift/impl/ClientRPCServiceImpl.java | 2 +- .../operator/process/AssignUniqueIdOperator.java | 142 +++++ .../operator/schema/SchemaQueryScanOperator.java | 12 +- .../operator/schema/source/ISchemaSource.java | 5 + .../schema/source/TableDeviceFetchSource.java | 21 + .../schema/source/TableDeviceQuerySource.java | 36 +- .../InformationSchemaContentSupplierFactory.java | 2 +- .../relational/MergeSortLeftJoinOperator.java | 163 +++++ .../aggregation/grouped/hash/HashStrategy.java | 2 +- .../execution/config/sys/pipe/ShowPipeTask.java | 2 +- .../plan/planner/TableOperatorGenerator.java | 53 ++ .../plan/planner/plan/node/PlanGraphPrinter.java | 10 + .../plan/planner/plan/node/PlanNodeType.java | 4 + .../plan/planner/plan/node/PlanVisitor.java | 5 + .../plan/planner/plan/node/write/InsertNode.java | 3 +- .../planner/plan/node/write/InsertRowNode.java | 3 - .../node/write/RelationalInsertTabletNode.java | 8 - .../PredicatePushIntoMetadataChecker.java | 7 +- .../fetcher/TableHeaderSchemaValidator.java | 4 + .../distribute/TableDistributedPlanGenerator.java | 58 +- .../iterative/rule/AggregationDecorrelation.java | 83 +++ .../iterative/rule/PruneAssignUniqueIdColumns.java | 47 ++ .../iterative/rule/RemoveRedundantExists.java | 104 +++ ...orrelatedDistinctAggregationWithProjection.java | 186 ++++++ ...elatedDistinctAggregationWithoutProjection.java | 167 +++++ ...mCorrelatedGlobalAggregationWithProjection.java | 318 ++++++++++ ...rrelatedGlobalAggregationWithoutProjection.java | 302 +++++++++ ...CorrelatedGroupedAggregationWithProjection.java | 252 ++++++++ ...relatedGroupedAggregationWithoutProjection.java | 233 +++++++ .../rule/TransformCorrelatedJoinToJoin.java | 99 +++ .../rule/TransformExistsApplyToCorrelatedJoin.java | 214 +++++++ .../relational/planner/node/AssignUniqueId.java | 97 +++ .../plan/relational/planner/node/Patterns.java | 27 +- .../planner/optimizations/JoinUtils.java | 249 +++++--- .../optimizations/LogicalOptimizeFactory.java | 40 +- .../optimizations/PlanNodeDecorrelator.java | 699 +++++++++++++++++++++ .../optimizations/PushPredicateIntoTableScan.java | 143 ++++- .../optimizations/UnaliasSymbolReferences.java | 13 + .../relational/planner/optimizations/Util.java | 19 + .../plan/relational/sql/ast/InsertRows.java | 3 - .../plan/statement/crud/InsertBaseStatement.java | 28 +- .../plan/statement/crud/InsertRowStatement.java | 3 - .../rescon/ISchemaRegionStatistics.java | 2 + .../rescon/MemSchemaRegionStatistics.java | 16 + .../attribute/DeviceAttributeStore.java | 48 +- .../attribute/IDeviceAttributeStore.java | 16 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 36 +- .../dataregion/memtable/AbstractMemTable.java | 24 +- .../dataregion/wal/utils/WALInsertNodeCache.java | 2 +- .../broker/SubscriptionPrefetchingQueue.java | 10 +- .../broker/SubscriptionPrefetchingTabletQueue.java | 2 +- .../broker/SubscriptionPrefetchingTsFileQueue.java | 2 +- .../db/subscription/event/SubscriptionEvent.java | 4 +- .../event/batch/SubscriptionPipeEventBatch.java | 2 +- .../event/batch/SubscriptionPipeEventBatches.java | 2 +- .../batch/SubscriptionPipeTabletEventBatch.java | 50 +- .../SubscriptionPipeTabletIterationSnapshot.java | 44 +- .../batch/SubscriptionPipeTsFileEventBatch.java | 2 +- .../event/pipe/SubscriptionPipeEmptyEvent.java | 2 +- .../event/pipe/SubscriptionPipeEvents.java | 2 +- .../pipe/SubscriptionPipeTabletBatchEvents.java | 11 +- .../pipe/SubscriptionPipeTsFileBatchEvents.java | 4 +- .../pipe/SubscriptionPipeTsFilePlainEvent.java | 2 +- .../SubscriptionConnectorSubtaskManager.java | 2 +- .../db/tools/schema/SRStatementGenerator.java | 2 + .../db/protocol/mqtt/PayloadFormatManagerTest.java | 2 +- .../execution/operator/OperatorMemoryTest.java | 2 - .../relational/analyzer/GroupByLargeDataTest.java | 179 ++++++ .../relational/planner/CorrelatedSubqueryTest.java | 153 +++++ ...ueryTest.java => UncorrelatedSubqueryTest.java} | 104 ++- .../planner/assertions/AssignUniqueIdMatcher.java} | 32 +- .../planner/assertions/PlanMatchPattern.java | 6 + .../conf/iotdb-system.properties.template | 25 +- .../apache/iotdb/commons/conf/CommonConfig.java | 34 + .../iotdb/commons/conf/CommonDescriptor.java | 16 + .../iotdb/commons/path/ExtendedPartialPath.java | 4 + .../agent/plugin/builtin/BuiltinPipePlugin.java | 5 + .../builtin/connector/opcda/OpcDaConnector.java} | 21 +- .../config/constant/PipeConnectorConstant.java | 8 +- .../connector/protocol/IoTDBAirGapConnector.java | 21 +- .../apache/iotdb/commons/schema/table/TsTable.java | 2 +- .../subscription/config/SubscriptionConfig.java | 21 + pom.xml | 8 +- 190 files changed, 7461 insertions(+), 788 deletions(-)
