This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch beyyes/fix_node_ordering in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 96978f8e0e028e6cb2b1b4c6763efc5a4aafcbc4 Merge: 33432be1949 7caaf2671a1 Author: Beyyes <[email protected]> AuthorDate: Wed Apr 2 12:30:31 2025 +0800 Merge branch 'master' into beyyes/fix_node_ordering .github/workflows/multi-language-client.yml | 10 +- .github/workflows/pipe-it.yml | 54 +- .github/workflows/vulnerability-check.yml | 7 +- dependencies.json | 11 +- .../server/CustomizedJsonPayloadFormatter.java | 8 +- .../java/org/apache/iotdb/mqtt/MQTTClient.java | 39 +- .../iotdb/udf/table/ExcludeColumnExample.java | 108 + .../org/apache/iotdb/udf/table/RepeatExample.java | 125 + .../org/apache/iotdb/udf/table/SplitExample.java | 114 + integration-test/pom.xml | 4 + .../example/relational/MyErrorTableFunction.java | 114 + .../udf/example/relational/MyExcludeColumn.java | 89 + .../udf/example/relational/MyRepeatWithIndex.java | 102 + .../example/relational/MyRepeatWithoutIndex.java | 99 + .../db/query/udf/example/relational/MySplit.java | 98 + .../iotdb/it/env/cluster/ClusterConstant.java | 1 + .../it/env/cluster/config/MppCommonConfig.java | 6 + .../it/env/cluster/config/MppDataNodeConfig.java | 12 + .../env/cluster/config/MppSharedCommonConfig.java | 7 + .../iotdb/it/env/cluster/node/DataNodeWrapper.java | 3 + .../it/env/remote/config/RemoteCommonConfig.java | 5 + .../it/env/remote/config/RemoteDataNodeConfig.java | 10 + .../org/apache/iotdb/itbase/env/CommonConfig.java | 2 + .../apache/iotdb/itbase/env/DataNodeConfig.java | 4 + .../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 17 +- .../{AbstractScript.java => AbstractScriptIT.java} | 2 +- .../apache/iotdb/cli/it/StartClientScriptIT.java | 2 +- .../it/cluster/IoTDBClusterNodeGetterIT.java | 2 +- ....java => IoTDBAutoRegionGroupExtension2IT.java} | 4 +- ...st.java => IoTDBPartitionTableAutoCleanIT.java} | 2 +- ...IoTDBRegionOperationReliabilityITFramework.java | 72 - .../IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java | 20 +- ...IoTDBRegionMigrateNormalITForIoTV2BatchIT.java} | 2 +- ...=> IoTDBRegionMigrateOtherForIoTV2BatchIT.java} | 2 +- ... IoTDBRegionMigrateNormalForIoTV2StreamIT.java} | 2 +- ...IoTDBRegionMigrateOtherITForIoTV2StreamIT.java} | 2 +- ...rCrashWhenRemoveRemotePeerForIoTV2BatchIT.java} | 2 +- ...RegionMigrateDataNodeCrashForIoTV2BatchIT.java} | 2 +- ...alCrashWhenDeleteLocalPeerForIoTV2BatchIT.java} | 2 +- ...lCrashWhenRemoveRemotePeerForIoTV2BatchIT.java} | 2 +- ...CrashWhenRemoveRemotePeerForIoTV2StreamIT.java} | 2 +- ...egionMigrateDataNodeCrashForIoTV2StreamIT.java} | 2 +- ...lCrashWhenDeleteLocalPeerForIoTV2StreamIT.java} | 2 +- ...CrashWhenRemoveRemotePeerForIoTV2StreamIT.java} | 2 +- ...oTDBRegionMigrateClusterCrashIoTV2BatchIT.java} | 2 +- ...BRegionMigrateConfigNodeCrashIoTV2BatchIT.java} | 2 +- ...TDBRegionMigrateClusterCrashIoTV2StreamIT.java} | 2 +- ...RegionMigrateConfigNodeCrashIoTV2StreamIT.java} | 2 +- .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 73 - .../org/apache/iotdb/db/it/IoTDBRestartIT.java | 3 +- .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 132 + .../it/auth/IoTDBClusterAuthorityRelationalIT.java | 1 - .../iotdb/db/it/auth/IoTDBRelationalAuthIT.java | 62 +- .../iotdb/db/it/auth/IoTDBSystemPermissionIT.java | 2 - .../it/schema/IoTDBDeleteAlignedTimeseriesIT.java | 42 + .../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 7 + .../org/apache/iotdb/db/it/utils/TestUtils.java | 9 +- .../pipe/it/dual/tablemodel/TableModelUtils.java | 14 +- .../manual/basic/IoTDBPipeDataSinkIT.java | 6 + .../manual/basic/IoTDBPipeExtractorIT.java | 7 + .../manual/basic/IoTDBPipeLifeCycleIT.java | 64 +- .../manual/basic/IoTDBPipePermissionIT.java | 314 ++ .../manual/basic/IoTDBPipeProtocolIT.java | 4 + .../manual/basic/IoTDBPipeSwitchStatusIT.java | 4 + .../tablemodel/manual/basic/IoTDBPipeSyntaxIT.java | 1 + .../manual/basic/IoTDBPipeWithLoadIT.java | 4 + .../manual/basic/IoTDBTablePatternFormatIT.java | 7 + .../manual/enhanced/IoTDBPipeAutoConflictIT.java | 4 + .../manual/enhanced/IoTDBPipeAutoDropIT.java | 2 + .../manual/enhanced/IoTDBPipeClusterIT.java | 13 + .../enhanced/IoTDBPipeConnectorCompressionIT.java | 12 +- .../enhanced/IoTDBPipeConnectorParallelIT.java | 2 + .../manual/enhanced/IoTDBPipeIdempotentIT.java | 13 + .../manual/enhanced/IoTDBPipeMetaIT.java | 4 + .../manual/enhanced/IoTDBPipeNullValueIT.java | 1 + .../treemodel/auto/basic/IoTDBPipeProcessorIT.java | 2 + .../pipe/it/single/IoTDBPipePermissionIT.java | 200 + .../it/db/it/IoTDBAuthenticationTableIT.java | 202 +- .../it/db/it/IoTDBFlushQueryTableIT.java | 30 +- .../relational/it/db/it/IoTDBInsertTableIT.java | 86 +- .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java | 18 +- .../relational/it/db/it/IoTDBWindowTVFIT.java | 229 ++ .../it/db/it/udf/IoTDBSQLFunctionManagementIT.java | 22 +- .../udf/IoTDBUserDefinedAggregateFunctionIT.java | 10 +- .../db/it/udf/IoTDBUserDefinedTableFunctionIT.java | 272 ++ .../relational/it/mqtt/IoTDBMQTTServiceIT.java | 132 + .../scalar/IoTDBFormatFunctionTableIT.java | 18 +- .../it/query/old/query/IoTDBArithmeticTableIT.java | 12 + .../query/old/query/IoTDBGreatestLeastTableIT.java | 2 +- .../it/query/recent/IoTDBColumnsMatchTableIT.java | 354 ++ .../it/query/recent/IoTDBMaintainAuthIT.java | 92 +- .../it/query/recent/IoTDBTableAggregationIT.java | 4 +- .../recent/IoTDBTableAggregationNonStreamIT.java | 56 + .../IoTDBCorrelatedExistsSubqueryIT.java | 432 ++ .../IoTDBUncorrelatedExistsSubqueryIT.java | 391 ++ .../relational/it/schema/IoTDBDatabaseIT.java | 108 +- .../iotdb/relational/it/schema/IoTDBTableIT.java | 129 +- .../it/session/IoTDBSessionRelationalIT.java | 186 +- .../iotdb/session/it/IoTDBSessionInsertNullIT.java | 28 + .../iotdb/session/it/IoTDBSessionSimpleIT.java | 48 - .../it/local/IoTDBSubscriptionBasicIT.java | 38 +- .../IoTDBDefaultPullConsumerDataSetIT.java | 28 +- ...TDBConsumer2With1TopicShareProcessTsfileIT.java | 14 +- .../IoTDBPathLooseDeviceTsfilePushConsumerIT.java | 10 +- .../IoTDBTimeLooseTsfilePushConsumerIT.java | 10 +- .../IoTDBTSPatternTsfilePushConsumerIT.java | 2 +- .../time/IoTDBRealTimeDBTsfilePushConsumerIT.java | 8 +- .../time/IoTDBTimeRangeDBTsfilePushConsumerIT.java | 22 +- .../apache/iotdb/tools/it/ExportDataTestIT.java | 4 +- .../apache/iotdb/tools/it/ExportSchemaTestIT.java | 4 +- .../apache/iotdb/tools/it/ExportTsFileTestIT.java | 40 +- .../apache/iotdb/tools/it/ImportDataTestIT.java | 4 +- .../apache/iotdb/tools/it/ImportSchemaTestIT.java | 4 +- .../src/test/resources/logback-test.xml | 2 +- ...sensusRetryWithIncreasingIntervalException.java | 11 +- .../iotdb/udf/api/relational/TableFunction.java | 83 +- .../iotdb/udf/api/relational/access/Record.java | 2 + .../relational/table/TableFunctionAnalysis.java | 113 + .../table/TableFunctionProcessorProvider.java | 43 + .../api/relational/table/argument/Argument.java | 44 +- .../relational/table/argument/DescribedSchema.java | 89 + .../relational/table/argument/ScalarArgument.java | 155 + .../relational/table/argument/TableArgument.java | 177 + .../processor/TableFunctionDataProcessor.java | 64 + .../processor/TableFunctionLeafProcessor.java | 26 +- .../specification/ParameterSpecification.java | 77 + .../ScalarParameterSpecification.java | 76 + .../specification/TableParameterSpecification.java | 98 + .../java/org/apache/iotdb/udf/api/type/Type.java | 38 + iotdb-client/cli/pom.xml | 4 + .../cli/src/assembly/resources/sbin/start-cli.sh | 4 - .../src/assembly/resources/tools/export-tsfile.bat | 62 - .../src/assembly/resources/tools/export-tsfile.sh | 59 - .../java/org/apache/iotdb/cli/AbstractCli.java | 55 +- .../org/apache/iotdb/tool/common/Constants.java | 33 +- .../org/apache/iotdb/tool/common/OptionsUtil.java | 167 +- .../apache/iotdb/tool/data/AbstractImportData.java | 10 +- .../org/apache/iotdb/tool/data/ExportData.java | 15 +- .../apache/iotdb/tool/data/ExportDataTable.java | 192 +- .../org/apache/iotdb/tool/data/ExportDataTree.java | 185 +- .../org/apache/iotdb/tool/data/ImportData.java | 8 +- .../apache/iotdb/tool/data/ImportDataTable.java | 23 +- .../org/apache/iotdb/tool/tsfile/ExportTsFile.java | 539 +-- .../subscription/AbstractSubscriptionTsFile.java | 74 + .../tool/tsfile/subscription/CommonParam.java | 212 + .../subscription/SubscriptionTableTsFile.java | 186 + .../subscription/SubscriptionTreeTsFile.java | 183 + iotdb-client/client-cpp/src/main/Session.cpp | 12 +- iotdb-client/client-py/iotdb/Session.py | 324 +- .../client-py/iotdb/template/MeasurementNode.py | 2 +- iotdb-client/client-py/iotdb/template/Template.py | 6 +- .../{TsFileConstant.py => tsfile_constant.py} | 0 .../tsfile/utils/{DateUtils.py => date_utils.py} | 0 .../iotdb/tsfile/utils/{Pair.py => pair.py} | 0 ...{ReadWriteIOUtils.py => read_write_io_utils.py} | 0 .../client-py/iotdb/tsfile/utils/tsblock_serde.py | 266 ++ iotdb-client/client-py/iotdb/utils/Field.py | 6 +- .../client-py/iotdb/utils/IoTDBRpcDataSet.py | 463 --- iotdb-client/client-py/iotdb/utils/NumpyTablet.py | 2 +- .../client-py/iotdb/utils/SessionDataSet.py | 48 +- iotdb-client/client-py/iotdb/utils/Tablet.py | 2 +- .../{IoTDBConnectionException.py => exception.py} | 20 + .../client-py/iotdb/utils/iotdb_rpc_dataset.py | 396 ++ iotdb-client/client-py/iotdb/utils/rpc_utils.py | 69 + iotdb-client/client-py/resources/pyproject.toml | 5 +- .../tests/integration/test_tablemodel_query.py | 456 +++ iotdb-client/jdbc/pom.xml | 55 +- .../main/java/org/apache/iotdb/jdbc/Constant.java | 4 +- ...ata.java => IoTDBAbstractDatabaseMetadata.java} | 4327 +++++++++----------- .../org/apache/iotdb/jdbc/IoTDBConnection.java | 40 +- .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 3496 ++-------------- .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 25 +- .../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 31 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 2 +- .../IoTDBRelationalDatabaseMetadata.java | 595 +++ .../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 + .../java/org/apache/iotdb/session/Session.java | 17 +- .../apache/iotdb/session/SessionConnection.java | 1019 ++--- .../base/AbstractSubscriptionConsumer.java | 29 +- .../SubscriptionTablePushConsumerBuilder.java | 3 +- .../apache/iotdb/session/util/CheckedSupplier.java | 32 - iotdb-core/ainode/.gitignore | 4 +- iotdb-core/ainode/{iotdb => ainode}/__init__.py | 0 .../{iotdb/ainode => ainode/core}/__init__.py | 0 .../ainode/{iotdb/ainode => ainode/core}/client.py | 42 +- .../ainode/{iotdb/ainode => ainode/core}/config.py | 22 +- .../{iotdb/ainode => ainode/core}/constant.py | 0 .../{iotdb/ainode => ainode/core}/exception.py | 2 +- .../{iotdb/ainode => ainode/core}/handler.py | 19 +- .../ainode/{iotdb/ainode => ainode/core}/log.py | 4 +- .../ainode => ainode/core}/manager/__init__.py | 0 .../core}/manager/cluster_manager.py | 4 +- .../core}/manager/inference_manager.py | 14 +- .../core}/manager/model_manager.py | 16 +- .../ainode => ainode/core}/model/__init__.py | 0 .../core}/model/built_in_model_factory.py | 8 +- .../ainode => ainode/core}/model/model_factory.py | 10 +- .../ainode => ainode/core}/model/model_storage.py | 15 +- .../ainode/{iotdb/ainode => ainode/core}/script.py | 16 +- .../{iotdb/ainode => ainode/core}/service.py | 8 +- .../{iotdb/ainode => ainode/core}/util/__init__.py | 0 .../ainode => ainode/core}/util/decorator.py | 0 .../{iotdb/ainode => ainode/core}/util/lock.py | 0 .../{iotdb/ainode => ainode/core}/util/serde.py | 6 +- .../{iotdb/ainode => ainode/core}/util/status.py | 6 +- iotdb-core/ainode/poetry.lock | 149 +- iotdb-core/ainode/pom.xml | 8 +- iotdb-core/ainode/pyproject.toml | 21 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 13 + .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 + iotdb-core/confignode/pom.xml | 4 + .../assembly/resources/sbin/stop-confignode.bat | 25 +- .../src/assembly/resources/sbin/stop-confignode.sh | 4 + .../confignode/client/CnToCnNodeRequestType.java | 2 +- .../client/async/CnToDnAsyncRequestType.java | 2 + .../CnToDnInternalServiceAsyncRequestManager.java | 11 + .../rpc/DataNodeAsyncRequestRPCHandler.java | 2 + .../client/sync/SyncConfigNodeClientPool.java | 6 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 56 +- .../confignode/conf/ConfigNodeDescriptor.java | 30 + .../consensus/request/ConfigPhysicalPlan.java | 8 + .../consensus/request/ConfigPhysicalPlanType.java | 2 + .../request/ConfigPhysicalPlanVisitor.java | 15 + .../request/write/model/UpdateModelInfoPlan.java | 4 + .../write/pipe/payload/PipeDeleteDevicesPlan.java | 44 +- .../request/write/table/AbstractTablePlan.java | 15 +- ...blePlan.java => SetTableColumnCommentPlan.java} | 41 +- ...ractTablePlan.java => SetTableCommentPlan.java} | 38 +- .../iotdb/confignode/manager/ClusterManager.java | 12 +- .../iotdb/confignode/manager/ConfigManager.java | 144 +- .../apache/iotdb/confignode/manager/IManager.java | 3 + .../iotdb/confignode/manager/ModelManager.java | 56 +- .../confignode/manager/PermissionManager.java | 6 + .../iotdb/confignode/manager/ProcedureManager.java | 56 +- .../iotdb/confignode/manager/load/LoadManager.java | 10 +- .../manager/load/balancer/PartitionBalancer.java | 8 +- .../manager/load/balancer/RouteBalancer.java | 154 +- .../manager/load/cache/AbstractLoadCache.java | 26 +- .../manager/load/cache/IFailureDetector.java | 28 +- .../confignode/manager/load/cache/LoadCache.java | 29 +- .../manager/load/cache/detector/FixedDetector.java | 58 + .../load/cache/detector/PhiAccrualDetector.java | 192 + .../load/cache/node/AINodeHeartbeatCache.java | 42 +- .../load/cache/node/ConfigNodeHeartbeatCache.java | 30 +- .../load/cache/node/DataNodeHeartbeatCache.java | 40 +- .../manager/load/cache/region/RegionCache.java | 37 +- .../load/cache/region/RegionGroupCache.java | 15 +- .../manager/load/service/EventService.java | 11 +- .../manager/load/service/HeartbeatService.java | 20 + .../manager/load/service/StatisticsService.java | 4 +- .../manager/load/service/TopologyService.java | 319 ++ .../load/subscriber/IClusterStatusSubscriber.java | 6 +- .../load/subscriber/NodeStatisticsChangeEvent.java | 3 +- .../iotdb/confignode/manager/node/NodeManager.java | 80 +- .../manager/partition/PartitionManager.java | 67 +- .../pipe/agent/task/PipeConfigNodeSubtask.java | 2 +- .../pipe/agent/task/PipeConfigNodeTaskAgent.java | 4 +- .../runtime/PipeLeaderChangeHandler.java | 12 - .../runtime/PipeRuntimeCoordinator.java | 12 - .../pipe/event/PipeConfigRegionSnapshotEvent.java | 14 +- .../pipe/event/PipeConfigRegionWritePlanEvent.java | 26 +- .../extractor/ConfigRegionListeningFilter.java | 9 +- .../pipe/extractor/IoTDBConfigRegionExtractor.java | 113 +- ...ConfigPhysicalPlanTablePatternParseVisitor.java | 64 +- ...nfigPhysicalPlanTablePrivilegeParseVisitor.java | 382 ++ ...eConfigPhysicalPlanTreePatternParseVisitor.java | 2 +- .../manager/pipe/metric/PipeConfigNodeMetrics.java | 9 + .../PipeConfigNodeRemainingTimeMetrics.java | 2 +- .../PipeConfigNodeRemainingTimeOperator.java | 2 +- .../PipeConfigNodeResourceMetrics.java | 2 +- .../{ => overview}/PipeProcedureMetrics.java | 2 +- .../metric/{ => overview}/PipeTaskInfoMetrics.java | 2 +- .../PipeTemporaryMetaInCoordinatorMetrics.java | 2 +- .../PipeConfigNodeReceiverMetrics.java | 2 +- .../PipeConfigRegionConnectorMetrics.java | 2 +- .../PipeConfigNodeListenerMetrics.java | 2 +- .../PipeConfigRegionExtractorMetrics.java | 2 +- .../receiver/protocol/IoTDBConfigNodeReceiver.java | 186 +- .../PipeConfigPhysicalPlanTSStatusVisitor.java | 26 +- .../resource/PipeConfigNodeResourceManager.java | 1 + .../PipeConfigNodeSnapshotResourceManager.java | 2 +- .../manager/schema/ClusterSchemaManager.java | 52 +- .../iotdb/confignode/persistence/AuthorInfo.java | 4 + .../iotdb/confignode/persistence/ModelInfo.java | 2 +- .../iotdb/confignode/persistence/TTLInfo.java | 2 +- .../persistence/executor/ConfigPlanExecutor.java | 16 +- .../confignode/persistence/node/NodeInfo.java | 7 + .../partition/DatabasePartitionTable.java | 14 +- .../confignode/persistence/pipe/PipeInfo.java | 2 +- .../persistence/pipe/PipePluginInfo.java | 51 +- .../persistence/schema/ClusterSchemaInfo.java | 59 + .../confignode/persistence/schema/ConfigMTree.java | 31 + .../procedure/PartitionTableAutoCleaner.java | 20 +- .../procedure/TimeoutExecutorThread.java | 2 +- .../procedure/env/ConfigNodeProcedureEnv.java | 33 +- .../procedure/env/RegionMaintainHandler.java | 8 - .../procedure/impl/StateMachineProcedure.java | 9 +- .../impl/node/RemoveConfigNodeProcedure.java | 8 +- .../impl/pipe/AbstractOperatePipeProcedureV2.java | 4 +- .../impl/pipe/task/AlterPipeProcedureV2.java | 67 +- .../impl/pipe/task/CreatePipeProcedureV2.java | 97 + .../impl/region/CreateRegionGroupsProcedure.java | 35 +- .../region/NotifyRegionMigrationProcedure.java | 137 + .../impl/region/ReconstructRegionProcedure.java | 33 - .../impl/region/RegionMigrateProcedure.java | 43 +- .../impl/schema/DeleteDatabaseProcedure.java | 4 +- .../impl/schema/UnsetTemplateProcedure.java | 100 +- ...eState.java => NotifyRegionMigrationState.java} | 6 +- .../procedure/state/RemoveConfigNodeState.java | 2 +- .../procedure/store/ProcedureFactory.java | 7 + .../confignode/procedure/store/ProcedureType.java | 1 + .../iotdb/confignode/service/ConfigNode.java | 44 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 27 +- .../request/ConfigPhysicalPlanSerDeTest.java | 33 +- .../manager/load/cache/RegionGroupCacheTest.java | 19 +- .../manager/load/cache/detector/DetectorTest.java | 180 + ...igPhysicalPlanTablePatternParseVisitorTest.java | 27 + ...figPhysicalPlanTreePatternParseVisitorTest.java | 2 +- .../common/request/IConsensusRequest.java | 5 + .../common/request/IndexedConsensusRequest.java | 8 +- .../iotdb/consensus/config/IoTConsensusConfig.java | 30 +- .../consensus/config/PipeConsensusConfig.java | 21 +- .../apache/iotdb/consensus/iot/IoTConsensus.java | 4 +- .../consensus/iot/client/DispatchLogHandler.java | 27 +- .../logdispatcher/IoTConsensusMemoryManager.java | 50 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 12 +- .../apache/iotdb/consensus/pipe/PipeConsensus.java | 80 +- .../consensus/pipe/PipeConsensusServerImpl.java | 25 +- .../pipe/consensuspipe/ConsensusPipeManager.java | 8 +- ...xManager.java => ReplicateProgressManager.java} | 8 +- .../pipe/metric/PipeConsensusSyncLagManager.java | 77 +- .../apache/iotdb/consensus/iot/StabilityTest.java | 49 +- iotdb-core/datanode/pom.xml | 10 +- .../src/assembly/resources/conf/datanode-env.sh | 3 + .../src/assembly/resources/sbin/stop-datanode.bat | 23 +- .../src/assembly/resources/sbin/stop-datanode.sh | 4 + .../ArithmeticBinaryColumnTransformer.ftl | 11 + .../org/apache/iotdb/db/auth/AuthorityChecker.java | 11 + .../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 684 ++++ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 412 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 348 +- .../db/consensus/DataRegionConsensusImpl.java | 15 +- .../PipeDataRegionConnectorConstructor.java | 4 + .../runtime/PipeSchemaRegionListenerManager.java | 2 +- .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 73 +- .../pipe/agent/task/connection/EnrichedDeque.java | 2 +- .../agent/task/connection/PipeEventCollector.java | 49 +- .../agent/task/stage/PipeTaskProcessorStage.java | 7 +- .../subtask/connector/PipeConnectorSubtask.java | 4 +- .../connector/PipeConnectorSubtaskManager.java | 2 +- .../PipeRealtimePriorityBlockingQueue.java | 53 +- .../subtask/processor/PipeProcessorSubtask.java | 22 +- .../client/IoTDBDataNodeAsyncClientManager.java | 10 +- .../evolvable/batch/PipeTabletEventBatch.java | 49 +- .../evolvable/batch/PipeTabletEventPlainBatch.java | 41 +- .../batch/PipeTabletEventTsFileBatch.java | 11 +- .../batch/PipeTransferBatchReqBuilder.java | 4 +- .../connector/protocol/opcda/OpcDaConnector.java | 138 + .../pipe/connector/protocol/opcda/OpcDaHeader.java | 200 + .../protocol/opcda/OpcDaServerHandle.java | 385 ++ .../connector/protocol/opcua/OpcUaConnector.java | 50 +- .../connector/protocol/opcua/OpcUaNameSpace.java | 17 +- .../pipeconsensus/PipeConsensusAsyncConnector.java | 16 +- .../pipeconsensus/PipeConsensusSyncConnector.java | 9 +- .../PipeConsensusTabletInsertionEventHandler.java | 8 +- .../PipeConsensusTsFileInsertionEventHandler.java | 8 +- .../PipeConsensusTransferBatchReqBuilder.java | 5 +- .../async/IoTDBDataRegionAsyncConnector.java | 252 +- .../async/handler/PipeTransferTsFileHandler.java | 77 +- .../protocol/writeback/WriteBackConnector.java | 96 +- .../util/builder/PipeTableModeTsFileBuilder.java | 5 +- .../sorter/PipeTableModelTabletEventSorter.java | 12 +- ....java => ReplicateProgressDataNodeManager.java} | 35 +- .../deletion/DeletionResourceManager.java | 10 +- .../deletion/persist/PageCacheDeletionBuffer.java | 18 +- .../db/pipe/event/UserDefinedEnrichedEvent.java | 14 +- .../db/pipe/event/common/PipeInsertionEvent.java | 17 +- .../common/deletion/PipeDeleteDataNodeEvent.java | 31 +- .../event/common/heartbeat/PipeHeartbeatEvent.java | 19 +- .../schema/PipeSchemaRegionSnapshotEvent.java | 16 +- .../schema/PipeSchemaRegionWritePlanEvent.java | 18 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 136 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 52 +- .../tablet/parser/TabletInsertionEventParser.java | 15 +- .../event/common/terminate/PipeTerminateEvent.java | 13 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 151 +- .../tsfile/parser/TsFileInsertionEventParser.java | 23 + .../parser/TsFileInsertionEventParserProvider.java | 81 +- .../query/TsFileInsertionEventQueryParser.java | 19 +- .../scan/TsFileInsertionEventScanParser.java | 28 +- .../table/TsFileInsertionEventTableParser.java | 127 +- ...ileInsertionEventTableParserTabletIterator.java | 439 +- .../db/pipe/event/realtime/PipeRealtimeEvent.java | 28 +- .../event/realtime/PipeRealtimeEventFactory.java | 92 +- .../dataregion/IoTDBDataRegionExtractor.java | 8 +- ...oricalDataRegionTsFileAndDeletionExtractor.java | 42 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 22 +- .../PipeRealtimeDataRegionHybridExtractor.java | 28 +- .../realtime/assigner/PipeDataRegionAssigner.java | 14 +- .../dataregion/realtime/epoch/TsFileEpoch.java | 2 +- .../realtime/epoch/TsFileEpochManager.java | 8 +- .../listener/PipeInsertionDataNodeListener.java | 18 +- .../matcher/CachedSchemaPatternMatcher.java | 40 +- .../realtime/matcher/PipeDataRegionMatcher.java | 3 + .../schemaregion/IoTDBSchemaRegionExtractor.java | 82 +- .../PipePlanTablePrivilegeParseVisitor.java | 81 + .../schemaregion/PipeStatementToPlanVisitor.java | 48 + .../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 15 + .../PipeDataNodeRemainingEventAndTimeMetrics.java | 2 +- .../PipeDataNodeRemainingEventAndTimeOperator.java | 2 +- .../{ => overview}/PipeHeartbeatEventMetrics.java | 2 +- .../metric/{ => overview}/PipeResourceMetrics.java | 2 +- .../overview/PipeTsFileToTabletsMetrics.java | 170 + .../PipeWALInsertNodeCacheMetrics.java | 2 +- .../{ => processor}/PipeProcessorMetrics.java | 2 +- .../PipeDataNodeReceiverMetrics.java | 2 +- .../PipeSchemaRegionConnectorMetrics.java | 2 +- .../PipeSchemaRegionExtractorMetrics.java | 2 +- .../PipeSchemaRegionListenerMetrics.java | 2 +- .../{ => sink}/PipeDataRegionConnectorMetrics.java | 2 +- .../metric/{ => source}/PipeAssignerMetrics.java | 2 +- .../{ => source}/PipeDataRegionEventCounter.java | 2 +- .../PipeDataRegionExtractorMetrics.java | 2 +- .../pipeconsensus/PipeConsensusProcessor.java | 43 +- .../pipeconsensus/PipeConsensusReceiver.java | 385 +- .../protocol/thrift/IoTDBDataNodeReceiver.java | 140 +- .../visitor/PipeStatementExceptionVisitor.java | 5 + ...leStatementDataTypeConvertExecutionVisitor.java | 1 + .../resource/memory/InsertNodeMemoryEstimator.java | 321 +- .../db/pipe/resource/memory/PipeMemoryBlock.java | 2 +- .../db/pipe/resource/memory/PipeMemoryManager.java | 121 +- .../pipe/resource/memory/PipeMemoryWeightUtil.java | 65 + .../pipe/resource/tsfile/PipeTsFileResource.java | 6 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 18 +- .../client/dn/AsyncTSStatusRPCHandler.java | 25 +- .../client/dn/DataNodeAsyncRequestRPCHandler.java | 20 +- .../client/dn/DataNodeIntraHeartbeatManager.java | 59 + .../db/protocol/mqtt/JSONPayloadFormatter.java | 9 +- .../db/protocol/mqtt/LinePayloadFormatter.java | 257 ++ .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 244 +- .../org/apache/iotdb/db/protocol/mqtt/Message.java | 58 +- .../iotdb/db/protocol/mqtt/PayloadFormatter.java | 6 + .../iotdb/db/protocol/mqtt/TableMessage.java | 144 + .../mqtt/{Message.java => TreeMessage.java} | 16 +- .../db/protocol/rest/impl/PingApiServiceImpl.java | 4 +- .../v1/handler/StatementConstructionHandler.java | 2 +- .../iotdb/db/protocol/session/SessionManager.java | 12 + .../protocol/thrift/impl/ClientRPCServiceImpl.java | 113 +- .../impl/DataNodeInternalRPCServiceImpl.java | 157 +- .../db/queryengine/execution/driver/Driver.java | 15 +- .../execution/exchange/SharedTsBlockQueue.java | 13 +- .../execution/exchange/sink/ISinkChannel.java | 2 + .../execution/exchange/sink/LocalSinkChannel.java | 21 +- .../execution/exchange/sink/ShuffleSinkHandle.java | 5 +- .../execution/exchange/sink/SinkChannel.java | 5 +- .../exchange/source/LocalSourceHandle.java | 12 +- .../execution/exchange/source/SourceHandle.java | 2 +- .../execution/executor/RegionWriteExecutor.java | 18 +- .../fragment/FragmentInstanceContext.java | 112 +- .../fragment/FragmentInstanceExecution.java | 18 +- .../fragment/FragmentInstanceManager.java | 26 +- .../execution/fragment/QueryStatistics.java | 10 + .../execution/memory/LocalMemoryManager.java | 5 +- .../queryengine/execution/memory/MemoryPool.java | 42 +- .../execution/operator/AbstractOperator.java | 5 +- ...owOperator.java => AssignUniqueIdOperator.java} | 77 +- .../operator/process/CollectOperator.java | 7 + .../operator/process/EnforceSingleRowOperator.java | 2 +- .../operator/process/ai/InferenceOperator.java | 42 +- .../process/function/PartitionRecognizer.java | 207 + .../function/TableFunctionLeafOperator.java | 111 + .../process/function/TableFunctionOperator.java | 281 ++ .../process/function/partition/PartitionCache.java | 112 + .../process/function/partition/PartitionState.java | 62 + .../operator/process/function/partition/Slice.java | 204 + .../join/SimpleNestedLoopCrossJoinOperator.java | 2 +- .../process/window/TableWindowOperator.java | 2 +- .../operator/schema/SchemaQueryScanOperator.java | 12 +- .../schema/source/DeviceAttributeUpdater.java | 10 +- .../schema/source/DeviceBlackListConstructor.java | 11 +- .../schema/source/DevicePredicateFilter.java | 13 +- .../schema/source/DevicePredicateHandler.java | 22 +- .../operator/schema/source/DeviceUpdater.java | 13 +- .../operator/schema/source/ISchemaSource.java | 5 + .../schema/source/SchemaSourceFactory.java | 4 +- .../schema/source/TableDeviceFetchSource.java | 21 + .../schema/source/TableDeviceQuerySource.java | 57 +- .../operator/sink/ShuffleHelperOperator.java | 2 +- .../relational/AbstractAggTableScanOperator.java | 17 +- .../relational/AbstractMergeSortJoinOperator.java | 4 +- .../relational/AbstractTableScanOperator.java | 2 +- .../InformationSchemaContentSupplierFactory.java | 15 +- .../relational/LastQueryAggTableScanOperator.java | 6 +- .../relational/MergeSortLeftJoinOperator.java | 163 + .../aggregation/CountAllAccumulator.java | 2 +- .../relational/aggregation/FirstByAccumulator.java | 11 +- .../relational/aggregation/LastByAccumulator.java | 6 +- .../relational/aggregation/MaxAccumulator.java | 6 +- .../relational/aggregation/MinAccumulator.java | 4 +- .../relational/aggregation/SumAccumulator.java | 4 +- .../relational/aggregation/TableAggregator.java | 9 +- .../aggregation/TableVarianceAccumulator.java | 4 +- .../aggregation/grouped/GroupedAggregator.java | 4 +- .../aggregation/grouped/GroupedAvgAccumulator.java | 2 +- .../grouped/GroupedExtremeAccumulator.java | 17 +- .../grouped/GroupedFirstAccumulator.java | 28 +- .../grouped/GroupedFirstByAccumulator.java | 20 +- .../grouped/GroupedLastAccumulator.java | 38 +- .../grouped/GroupedLastByAccumulator.java | 20 +- .../aggregation/grouped/GroupedMaxAccumulator.java | 16 +- .../grouped/GroupedMaxMinByBaseAccumulator.java | 30 +- .../aggregation/grouped/GroupedMinAccumulator.java | 16 +- .../aggregation/grouped/GroupedSumAccumulator.java | 3 +- .../grouped/GroupedVarianceAccumulator.java | 4 +- .../grouped/StreamingAggregationOperator.java | 32 +- .../grouped/StreamingHashAggregationOperator.java | 35 +- .../aggregation/grouped/hash/HashStrategy.java | 2 +- .../execution/schedule/AbstractDriverThread.java | 27 +- .../execution/schedule/DriverScheduler.java | 8 +- .../metric/SeriesScanCostMetricSet.java | 40 +- .../iotdb/db/queryengine/plan/ClusterTopology.java | 175 + .../iotdb/db/queryengine/plan/Coordinator.java | 12 +- .../db/queryengine/plan/analyze/AnalyzeUtils.java | 2 - .../queryengine/plan/analyze/AnalyzeVisitor.java | 84 +- .../analyze/cache/partition/PartitionCache.java | 16 +- .../cache/schema/DataNodeDevicePathCache.java | 23 +- .../schema/dualkeycache/impl/DualKeyCacheImpl.java | 9 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 756 +++- .../analyze/load/LoadTsFileTableSchemaCache.java | 30 +- .../load/LoadTsFileToTableModelAnalyzer.java | 226 - .../load/LoadTsFileToTreeModelAnalyzer.java | 187 - .../analyze/load/LoadTsFileTreeSchemaCache.java | 6 +- .../load/TreeSchemaAutoCreatorAndVerifier.java | 4 +- .../analyze/lock/DataNodeSchemaLockManager.java | 1 + .../plan/analyze/lock/SchemaLockType.java | 15 +- .../analyze/schema/AutoCreateSchemaExecutor.java | 6 +- .../analyze/schema/ClusterSchemaFetchExecutor.java | 11 +- .../plan/analyze/schema/ClusterSchemaFetcher.java | 6 +- .../plan/analyze/schema/SchemaValidator.java | 2 +- .../plan/analyze/schema/TemplateSchemaFetcher.java | 11 +- .../queryengine/plan/execution/QueryExecution.java | 9 +- .../execution/config/TableConfigTaskVisitor.java | 313 +- .../execution/config/TreeConfigTaskVisitor.java | 73 +- .../config/executor/ClusterConfigTaskExecutor.java | 261 +- .../config/executor/IConfigTaskExecutor.java | 43 +- .../config/metadata/CreatePipePluginTask.java | 2 + .../config/metadata/ShowFunctionsTask.java | 6 + .../metadata/{model => ai}/CreateModelTask.java | 2 +- .../config/metadata/ai/CreateTrainingTask.java | 108 + .../metadata/{model => ai}/DropModelTask.java | 2 +- .../metadata/{model => ai}/ShowModelsTask.java | 2 +- .../AlterTableCommentColumnTask.java} | 28 +- .../AlterTableCommentTableTask.java} | 23 +- .../relational/DescribeTableDetailsTask.java | 10 + .../metadata/relational/ShowTablesDetailsTask.java | 8 + .../execution/config/sys/pipe/AlterPipeTask.java | 3 +- .../execution/config/sys/pipe/ShowPipeTask.java | 2 +- .../db/queryengine/plan/parser/ASTVisitor.java | 69 +- .../plan/parser/StatementGenerator.java | 4 +- .../plan/planner/LocalExecutionPlanContext.java | 3 +- .../plan/planner/LocalExecutionPlanner.java | 124 +- .../plan/planner/LogicalPlanBuilder.java | 1 + .../plan/planner/LogicalPlanVisitor.java | 6 +- .../plan/planner/OperatorTreeGenerator.java | 1 + .../plan/planner/TableOperatorGenerator.java | 181 +- .../SimpleFragmentParallelPlanner.java | 121 +- .../distribution/WriteFragmentParallelPlanner.java | 10 +- .../ReplicaSetUnreachableException.java} | 21 +- .../exceptions/RootFIPlacementException.java} | 25 +- .../plan/AbstractFragmentParallelPlanner.java | 168 + .../plan/planner/plan/FragmentInstance.java | 2 +- .../plan/planner/plan/PlanFragment.java | 22 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 84 + .../plan/planner/plan/node/PlanNodeType.java | 17 +- .../plan/planner/plan/node/PlanVisitor.java | 20 + .../plan/node/process/AI/InferenceNode.java | 18 +- .../plan/planner/plan/node/write/InsertNode.java | 6 +- .../planner/plan/node/write/InsertRowNode.java | 3 - .../plan/node/write/RelationalInsertRowNode.java | 5 - .../node/write/RelationalInsertTabletNode.java | 8 - .../plan/relational/analyzer/Analysis.java | 27 +- .../relational/analyzer/ExpressionAnalyzer.java | 6 + .../relational/analyzer/StatementAnalyzer.java | 1443 ++++++- .../analyzer/StatementAnalyzerFactory.java | 4 + .../PredicatePushIntoMetadataChecker.java | 7 +- .../analyzer/tablefunction/ArgumentAnalysis.java} | 39 +- .../analyzer/tablefunction/ArgumentsAnalysis.java | 52 + .../tablefunction/TableArgumentAnalysis.java | 152 + .../TableFunctionInvocationAnalysis.java | 75 + .../relational/metadata/AlignedDeviceEntry.java | 20 +- .../plan/relational/metadata/DeviceEntry.java | 26 +- .../plan/relational/metadata/Metadata.java | 3 + .../metadata/NonAlignedAlignedDeviceEntry.java | 20 +- .../relational/metadata/TableMetadataImpl.java | 15 +- .../plan/relational/metadata/TableSchema.java | 2 +- .../fetcher/DeviceInCacheFilterVisitor.java | 4 +- .../metadata/fetcher/SchemaPredicateUtil.java | 10 +- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 34 +- .../fetcher/TableDeviceSchemaValidator.java | 6 +- .../fetcher/TableHeaderSchemaValidator.java | 63 +- .../fetcher/cache/TableAttributeSchema.java | 6 +- .../fetcher/cache/TableDeviceLastCache.java | 5 +- .../fetcher/cache/TableDeviceSchemaCache.java | 30 +- .../planner/DataOrganizationSpecification.java | 106 + .../plan/relational/planner/QueryPlanner.java | 20 + .../plan/relational/planner/RelationPlanner.java | 141 +- .../relational/planner/TableLogicalPlanner.java | 15 +- .../planner/distribute/AddExchangeNodes.java | 36 +- .../distribute/TableDistributedPlanGenerator.java | 401 +- .../distribute/TableDistributedPlanner.java | 12 +- .../distribute/TableModelQueryFragmentPlanner.java | 170 +- .../TableModelTypeProviderExtractor.java | 7 + .../plan/relational/planner/ir/IrUtils.java | 13 +- .../iterative/rule/AggregationDecorrelation.java | 83 + .../rule/ImplementTableFunctionSource.java | 135 + .../rule/PruneAssignUniqueIdColumns.java} | 33 +- .../rule/PruneTableFunctionProcessorColumns.java | 88 + .../PruneTableFunctionProcessorSourceColumns.java | 98 + .../iterative/rule/RemoveRedundantExists.java | 104 + ...orrelatedDistinctAggregationWithProjection.java | 186 + ...elatedDistinctAggregationWithoutProjection.java | 167 + ...mCorrelatedGlobalAggregationWithProjection.java | 318 ++ ...rrelatedGlobalAggregationWithoutProjection.java | 302 ++ ...CorrelatedGroupedAggregationWithProjection.java | 252 ++ ...relatedGroupedAggregationWithoutProjection.java | 233 ++ .../rule/TransformCorrelatedJoinToJoin.java | 99 + .../rule/TransformExistsApplyToCorrelatedJoin.java | 214 + .../relational/planner/node/AssignUniqueId.java | 97 + .../plan/relational/planner/node/ExchangeNode.java | 6 +- .../plan/relational/planner/node/GroupNode.java | 104 + .../plan/relational/planner/node/OutputNode.java | 2 + .../plan/relational/planner/node/Patterns.java | 40 +- .../relational/planner/node/TableFunctionNode.java | 372 ++ .../planner/node/TableFunctionProcessorNode.java | 306 ++ .../planner/optimizations/JoinUtils.java | 249 +- .../optimizations/LogicalOptimizeFactory.java | 49 +- .../planner/optimizations/ParallelizeGrouping.java | 266 ++ .../optimizations/PlanNodeDecorrelator.java | 699 ++++ .../PushLimitOffsetIntoTableScan.java | 17 + .../optimizations/PushPredicateIntoTableScan.java | 159 +- .../planner/optimizations/SymbolMapper.java | 7 + .../TransformAggregationToStreamable.java | 22 + .../optimizations/TransformSortToStreamSort.java | 6 + .../optimizations/UnaliasSymbolReferences.java | 143 + .../relational/planner/optimizations/Util.java | 19 + .../plan/relational/security/AccessControl.java | 18 +- .../relational/security/AccessControlImpl.java | 61 +- .../relational/security/AllowAllAccessControl.java | 11 +- .../relational/security/ITableAuthChecker.java | 3 + .../relational/security/ITableAuthCheckerImpl.java | 18 + .../relational/security/TableModelPrivilege.java | 5 - .../sql/ast/AbstractQueryDeviceWithCache.java | 33 +- .../relational/sql/ast/AbstractTraverseDevice.java | 11 +- .../plan/relational/sql/ast/AstVisitor.java | 37 + .../plan/relational/sql/ast/ColumnDefinition.java | 27 +- .../plan/relational/sql/ast/Columns.java | 96 + .../plan/relational/sql/ast/CreateTable.java | 8 + .../plan/relational/sql/ast/CreateTraining.java | 156 + .../plan/relational/sql/ast/DeleteDevice.java | 19 +- .../plan/relational/sql/ast/InsertRows.java | 3 - .../plan/relational/sql/ast/LoadTsFile.java | 101 +- .../plan/relational/sql/ast/SetColumnComment.java | 116 + .../plan/relational/sql/ast/SetProperties.java | 12 - .../plan/relational/sql/ast/SetTableComment.java | 98 + .../plan/relational/sql/ast/ShowDevice.java | 10 +- .../plan/relational/sql/ast/ShowModels.java} | 51 +- .../sql/ast/ShowQueriesStatement.java} | 24 +- .../plan/relational/sql/ast/ShowRegions.java | 43 +- .../plan/relational/sql/ast/SingleColumn.java | 21 + .../relational/sql/ast/TableFunctionArgument.java | 92 + .../sql/ast/TableFunctionInvocation.java | 92 + .../sql/ast/TableFunctionTableArgument.java | 119 + .../queryengine/plan/relational/sql/ast/Trim.java | 2 +- .../plan/relational/sql/parser/AstBuilder.java | 343 +- .../plan/relational/sql/rewrite/ShowRewrite.java | 26 +- .../sql/rewrite/StatementRewriteFactory.java | 6 +- .../plan/relational/sql/util/AstUtil.java | 22 + .../relational/sql/util/ExpressionFormatter.java | 6 + .../plan/relational/sql/util/SqlFormatter.java | 117 +- .../plan/scheduler/AsyncSendPlanNodeHandler.java | 2 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 59 +- .../scheduler/load/LoadTsFileDispatcherImpl.java | 11 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 118 +- .../plan/statement/StatementVisitor.java | 5 + .../plan/statement/crud/InsertBaseStatement.java | 74 +- .../crud/InsertMultiTabletsStatement.java | 16 + .../plan/statement/crud/InsertRowStatement.java | 16 +- .../crud/InsertRowsOfOneDeviceStatement.java | 16 + .../plan/statement/crud/InsertRowsStatement.java | 16 + .../plan/statement/crud/InsertTabletStatement.java | 18 + .../plan/statement/crud/LoadTsFileStatement.java | 106 +- .../plan/statement/crud/QueryStatement.java | 9 + .../metadata/CountTimeSlotListStatement.java | 7 + .../statement/metadata/GetRegionIdStatement.java | 7 + .../metadata/GetSeriesSlotListStatement.java | 7 + .../metadata/GetTimeSlotListStatement.java | 7 + .../metadata/RemoveConfigNodeStatement.java | 9 +- .../metadata/RemoveDataNodeStatement.java | 9 +- .../statement/metadata/ShowClusterIdStatement.java | 7 + .../statement/metadata/ShowClusterStatement.java | 9 +- .../metadata/ShowConfigNodesStatement.java | 4 +- .../statement/metadata/ShowDataNodesStatement.java | 4 +- .../statement/metadata/ShowRegionStatement.java | 33 +- .../metadata/model/CreateTrainingStatement.java | 140 + .../metadata/model/ShowAINodesStatement.java | 7 + .../metadata/pipe/AlterPipeStatement.java | 9 + .../metadata/pipe/CreatePipePluginStatement.java | 10 + .../metadata/region/ExtendRegionStatement.java | 9 +- .../metadata/region/MigrateRegionStatement.java | 9 +- .../region/ReconstructRegionStatement.java | 9 +- .../metadata/region/RemoveRegionStatement.java | 9 +- .../plan/statement/sys/KillQueryStatement.java | 9 +- .../plan/statement/sys/ShowQueriesStatement.java | 9 +- .../statement/sys/TestConnectionStatement.java | 3 +- .../unary/scalar/FormatColumnTransformer.java | 4 +- .../metric/SchemaEngineCachedMetric.java | 5 +- .../rescon/ISchemaRegionStatistics.java | 2 + .../rescon/MemSchemaEngineStatistics.java | 41 +- .../rescon/MemSchemaRegionStatistics.java | 16 + .../attribute/DeviceAttributeStore.java | 48 +- .../attribute/IDeviceAttributeStore.java | 16 +- .../attribute/update/UpdateDetailContainer.java | 2 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 65 +- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 2 +- .../ReleaseFlushStrategySizeBasedImpl.java | 6 +- .../read/resp/info/impl/ShowDevicesResult.java | 2 +- .../schemaregion/utils/ResourceByPathUtils.java | 12 +- .../db/schemaengine/table/DataNodeTableCache.java | 28 +- .../schemaengine/table/InformationSchemaUtils.java | 2 + .../template/TemplateInternalRPCUpdateType.java | 4 +- .../java/org/apache/iotdb/db/service/DataNode.java | 66 +- .../iotdb/db/service/DataNodeShutdownHook.java | 22 +- .../{RPCService.java => ExternalRPCService.java} | 6 +- ...viceMBean.java => ExternalRPCServiceMBean.java} | 2 +- .../org/apache/iotdb/db/service/MQTTService.java | 2 + .../iotdb/db/service/RegionMigrateService.java | 67 + .../db/service/metrics/CompactionMetrics.java | 2 +- .../iotdb/db/service/metrics/WritingMetrics.java | 2 +- .../metrics/memory/ConsensusMemoryMetrics.java | 50 +- .../metrics/memory/GlobalMemoryMetrics.java | 82 +- .../metrics/memory/OffHeapMemoryMetrics.java | 102 + .../metrics/memory/QueryEngineMemoryMetrics.java | 281 +- .../metrics/memory/SchemaEngineMemoryMetrics.java | 159 +- .../metrics/memory/StorageEngineMemoryMetrics.java | 361 +- .../metrics/memory/StreamEngineMemoryMetrics.java | 50 +- .../iotdb/db/storageengine/StorageEngine.java | 24 +- .../db/storageengine/buffer/BloomFilterCache.java | 27 +- .../iotdb/db/storageengine/buffer/ChunkCache.java | 27 +- .../buffer/TimeSeriesMetadataCache.java | 50 +- .../db/storageengine/dataregion/DataRegion.java | 87 +- .../storageengine/dataregion/DataRegionInfo.java | 6 +- .../dataregion/VirtualDataRegion.java | 2 +- .../execute/task/CrossSpaceCompactionTask.java | 2 +- .../execute/task/InnerSpaceCompactionTask.java | 2 +- .../task/InsertionCrossSpaceCompactionTask.java | 10 +- .../task/RepairUnsortedFileCompactionTask.java | 2 +- .../compaction/execute/utils/CompactionUtils.java | 12 +- .../execute/utils/MultiTsFileDeviceIterator.java | 5 +- .../ReadChunkAlignedSeriesCompactionExecutor.java | 5 +- .../writer/AbstractCrossCompactionWriter.java | 25 +- .../utils/writer/FastCrossCompactionWriter.java | 12 + .../dataregion/memtable/AbstractMemTable.java | 87 +- .../memtable/AbstractWritableMemChunk.java | 217 + .../memtable/AlignedReadOnlyMemChunk.java | 402 +- .../memtable/AlignedWritableMemChunk.java | 350 +- .../memtable/AlignedWritableMemChunkGroup.java | 14 +- .../dataregion/memtable/IMemTable.java | 2 +- .../dataregion/memtable/IWritableMemChunk.java | 31 - .../memtable/IWritableMemChunkGroup.java | 2 +- .../dataregion/memtable/ReadOnlyMemChunk.java | 160 +- .../dataregion/memtable/TsFileProcessor.java | 43 +- .../dataregion/memtable/WritableMemChunk.java | 466 +-- .../dataregion/memtable/WritableMemChunkGroup.java | 18 +- .../read/reader/chunk/MemAlignedChunkReader.java | 226 +- .../read/reader/chunk/MemAlignedPageReader.java | 22 +- .../read/reader/chunk/MemChunkReader.java | 111 +- .../read/reader/chunk/MemPageReader.java | 23 +- .../dataregion/tsfile/TsFileResource.java | 85 +- .../dataregion/tsfile/TsFileResourceBlockType.java | 3 + .../tsfile/timeindex/ArrayDeviceTimeIndex.java | 15 +- .../dataregion/tsfile/timeindex/FileTimeIndex.java | 2 +- .../dataregion/wal/buffer/IWALBuffer.java | 10 + .../dataregion/wal/buffer/WALBuffer.java | 13 + .../storageengine/dataregion/wal/node/WALNode.java | 8 +- .../wal/recover/file/TsFilePlanRedoer.java | 27 +- .../wal/utils/MemoryControlledWALEntryQueue.java | 9 +- .../dataregion/wal/utils/WALEntryHandler.java | 4 +- .../dataregion/wal/utils/WALInsertNodeCache.java | 16 +- .../load/active/ActiveLoadTsFileLoader.java | 102 +- .../load/config/LoadTsFileConfigurator.java | 42 +- ...ertedInsertTabletStatementExceptionVisitor.java | 5 + ...vertedInsertTabletStatementTSStatusVisitor.java | 7 + ...leStatementDataTypeConvertExecutionVisitor.java | 16 +- ...eeStatementDataTypeConvertExecutionVisitor.java | 200 +- .../storageengine/load/disk/ILoadDiskSelector.java | 66 + .../InheritSystemMultiDisksStrategySelector.java | 58 + .../db/storageengine/load/disk/MinIOSelector.java | 112 + .../memory/LoadTsFileDataCacheMemoryBlock.java | 4 +- ...MemoryBlock.java => LoadTsFileMemoryBlock.java} | 15 +- .../load/memory/LoadTsFileMemoryManager.java | 38 +- .../load/metrics/LoadTsFileCostMetricsSet.java | 31 +- .../load/metrics/LoadTsFileMemMetricSet.java | 6 +- .../db/storageengine/rescon/disk/TierManager.java | 2 +- .../rescon/memory/PrimitiveArrayManager.java | 16 +- .../db/storageengine/rescon/memory/SystemInfo.java | 187 +- .../rescon/memory/TimePartitionManager.java | 31 +- .../rescon/memory/TsFileResourceManager.java | 40 +- .../agent/SubscriptionBrokerAgent.java | 10 +- .../db/subscription/broker/SubscriptionBroker.java | 25 +- .../broker/SubscriptionPrefetchingQueue.java | 21 +- .../broker/SubscriptionPrefetchingTabletQueue.java | 12 +- .../broker/SubscriptionPrefetchingTsFileQueue.java | 12 +- .../db/subscription/event/SubscriptionEvent.java | 4 +- .../event/batch/SubscriptionPipeEventBatch.java | 2 +- .../event/batch/SubscriptionPipeEventBatches.java | 2 +- .../batch/SubscriptionPipeTabletEventBatch.java | 50 +- .../SubscriptionPipeTabletIterationSnapshot.java | 44 +- .../batch/SubscriptionPipeTsFileEventBatch.java | 7 +- .../event/cache/SubscriptionPollResponseCache.java | 11 + .../event/pipe/SubscriptionPipeEmptyEvent.java | 2 +- .../event/pipe/SubscriptionPipeEvents.java | 2 +- .../pipe/SubscriptionPipeTabletBatchEvents.java | 11 +- .../pipe/SubscriptionPipeTsFileBatchEvents.java | 20 +- .../pipe/SubscriptionPipeTsFilePlainEvent.java | 2 +- .../SubscriptionDataNodeResourceManager.java} | 37 +- .../resource/log/SubscriptionLogManager.java} | 23 +- .../resource/log/SubscriptionLogStatus.java | 82 + .../SubscriptionConnectorSubtaskManager.java | 2 +- .../db/tools/schema/SRStatementGenerator.java | 33 +- .../org/apache/iotdb/db/utils/DateTimeUtils.java | 35 +- .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 10 +- .../db/utils/datastructure/AlignedTVList.java | 557 ++- .../iotdb/db/utils/datastructure/BinaryTVList.java | 42 +- .../db/utils/datastructure/BooleanTVList.java | 42 +- .../iotdb/db/utils/datastructure/DoubleTVList.java | 42 +- .../datastructure/FixedPriorityBlockingQueue.java | 2 +- .../iotdb/db/utils/datastructure/FloatTVList.java | 42 +- .../iotdb/db/utils/datastructure/IntTVList.java | 42 +- .../iotdb/db/utils/datastructure/LongTVList.java | 42 +- .../datastructure/MemPointIterator.java} | 14 +- .../datastructure/MemPointIteratorFactory.java | 381 ++ .../MergeSortAlignedTVListIterator.java | 242 -- .../MergeSortMultiAlignedTVListIterator.java | 176 + .../MergeSortMultiTVListIterator.java | 83 + .../datastructure/MergeSortTvListIterator.java | 162 - .../datastructure/MultiAlignedTVListIterator.java | 236 ++ .../utils/datastructure/MultiTVListIterator.java | 179 + .../OrderedMultiAlignedTVListIterator.java | 116 + .../datastructure/OrderedMultiTVListIterator.java | 58 + .../iotdb/db/utils/datastructure/TVList.java | 306 +- ....apache.iotdb.db.protocol.mqtt.PayloadFormatter | 1 + .../db/pipe/consensus/DeletionResourceTest.java | 7 +- .../pipe/event/TsFileInsertionEventParserTest.java | 5 +- .../db/pipe/extractor/PipeRealtimeExtractTest.java | 5 + .../pattern/CachedSchemaPatternMatcherTest.java | 53 +- .../db/protocol/mqtt/JSONPayloadFormatterTest.java | 8 +- .../db/protocol/mqtt/LinePayloadFormatterTest.java | 91 + .../db/protocol/rest/IoTDBRestServiceTest.java | 6 +- .../execution/exchange/LocalSinkChannelTest.java | 7 +- .../exchange/MPPDataExchangeManagerTest.java | 7 +- .../execution/exchange/SharedTsBlockQueueTest.java | 4 +- .../execution/exchange/ShuffleSinkHandleTest.java | 4 +- .../execution/exchange/SinkChannelTest.java | 7 +- .../execution/exchange/SourceHandleTest.java | 15 +- .../execution/memory/MemoryPoolTest.java | 6 +- .../execution/operator/OperatorMemoryTest.java | 2 - .../process/tvf/TableFunctionOperatorTest.java | 304 ++ .../db/queryengine/plan/function/Exclude.java | 89 + .../iotdb/db/queryengine/plan/function/Repeat.java | 106 + .../iotdb/db/queryengine/plan/function/Split.java | 98 + .../plan/parser/StatementGeneratorTest.java | 2 +- .../analyzer/AggregationCornerCaseTest.java | 502 +++ .../plan/relational/analyzer/AggregationTest.java | 305 +- .../plan/relational/analyzer/AnalyzerTest.java | 156 +- .../plan/relational/analyzer/DistinctTest.java | 48 +- .../plan/relational/analyzer/JoinTest.java | 94 +- .../analyzer/LimitOffsetPushDownTest.java | 183 +- .../analyzer/MockTableModelDataPartition.java | 63 +- .../plan/relational/analyzer/SortTest.java | 288 +- .../plan/relational/analyzer/SubQueryTest.java | 44 +- .../plan/relational/analyzer/TSBSMetadata.java | 28 +- .../relational/analyzer/TableFunctionTest.java | 329 ++ .../plan/relational/analyzer/TestMatadata.java | 32 +- .../plan/relational/analyzer/TestUtils.java | 26 + .../plan/relational/analyzer/TreeViewTest.java | 72 +- .../fetcher/cache/TableDeviceSchemaCacheTest.java | 21 +- .../relational/planner/CorrelatedSubqueryTest.java | 153 + .../plan/relational/planner/ExampleTest.java | 34 +- .../plan/relational/planner/PlanTester.java | 9 +- ...ueryTest.java => UncorrelatedSubqueryTest.java} | 268 +- ...NodeMatcher.java => AssignUniqueIdMatcher.java} | 27 +- .../planner/assertions/ExchangeNodeMatcher.java | 8 +- .../planner/assertions/PlanMatchPattern.java | 35 +- .../planner/assertions/PlanMatchingVisitor.java | 12 +- .../planner/assertions/SpecificationProvider.java | 79 + .../assertions/TableFunctionProcessorMatcher.java | 303 ++ .../relational/sql/parser/AuthorStatementTest.java | 5 - .../compaction/CompactionWithMinTimestampTest.java | 136 + .../compaction/CompactionWorkerTest.java | 16 +- .../FastCrossCompactionPerformerTest.java | 83 +- .../cross/CrossSpaceCompactionSelectorTest.java | 5 +- ...atchedAlignedSeriesReadChunkCompactionTest.java | 2 +- .../CompactionTableModelTestFileWriter.java | 14 + .../compaction/utils/CompactionClearUtils.java | 2 +- .../compaction/utils/CompactionTaskQueueTest.java | 13 +- .../dataregion/memtable/PrimitiveMemTableTest.java | 88 + .../dataregion/memtable/TsFileProcessorTest.java | 36 +- .../reader/chunk/AlignedMemPageReaderTest.java | 8 +- .../reader/chunk/MemAlignedChunkLoaderTest.java | 12 +- .../read/reader/chunk/MemChunkLoaderTest.java | 58 +- .../read/reader/chunk/MemPageReaderTest.java | 4 +- .../wal/recover/file/TsFilePlanRedoerTest.java | 69 + .../rescon/memory/ResourceManagerTest.java | 6 +- .../rescon/memory/TimePartitionManagerTest.java | 8 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 5 +- .../datastructure/PrimitiveArrayManagerTest.java | 8 +- .../datanode/src/test/resources/logback-test.xml | 2 +- .../apache/iotdb/metrics/utils/FileStoreUtils.java | 2 +- .../conf/iotdb-system.properties.template | 49 +- .../commons/auth/authorizer/BasicAuthorizer.java | 35 +- .../iotdb/commons/auth/authorizer/IAuthorizer.java | 10 +- .../iotdb/commons/auth/entity/PrivilegeType.java | 2 +- .../org/apache/iotdb/commons/auth/entity/Role.java | 4 + .../iotdb/commons/auth/user/BasicUserManager.java | 12 +- .../iotdb/commons/client/ClientPoolFactory.java | 1 + .../iotdb/commons/client/ainode/AINodeClient.java | 13 + .../async/AsyncPipeDataTransferServiceClient.java | 14 + .../client/request/AsyncRequestManager.java | 10 +- .../DataNodeIntraHeartbeatRequestManager.java | 44 + .../client/request/TestConnectionUtils.java | 3 +- .../iotdb/commons/concurrent/ThreadName.java | 3 + .../apache/iotdb/commons/conf/CommonConfig.java | 172 +- .../iotdb/commons/conf/CommonDescriptor.java | 106 +- .../apache/iotdb/commons/conf/IoTDBConstant.java | 1 + .../consensus/index/impl/HybridProgressIndex.java | 4 +- .../commons/memory/AtomicLongMemoryBlock.java | 152 + .../apache/iotdb/commons/memory/IMemoryBlock.java | 135 + .../MemoryBlockType.java} | 14 +- .../RetryUtils.java => memory/MemoryConfig.java} | 35 +- .../iotdb/commons/memory/MemoryException.java | 64 + .../apache/iotdb/commons/memory/MemoryManager.java | 523 +++ .../iotdb/commons/model/ModelInformation.java | 20 + .../apache/iotdb/commons/model/ModelStatus.java | 1 + .../commons/partition/DataPartitionTable.java | 10 +- .../iotdb/commons/partition/ExecutorType.java | 8 +- .../iotdb/commons/partition/QueryExecutor.java | 8 +- .../commons/partition/SeriesPartitionTable.java | 20 +- .../iotdb/commons/partition/StorageExecutor.java | 10 +- .../iotdb/commons/path/ExtendedPartialPath.java | 4 + .../agent/plugin/builtin/BuiltinPipePlugin.java | 6 +- .../builtin/connector/opcda/OpcDaConnector.java} | 23 +- .../commons/pipe/agent/task/PipeTaskAgent.java | 2 +- .../task/subtask/PipeAbstractConnectorSubtask.java | 4 +- .../agent/task/subtask/PipeReportableSubtask.java | 13 +- .../iotdb/commons/pipe/config/PipeConfig.java | 67 +- .../config/constant/PipeConnectorConstant.java | 12 +- .../config/constant/PipeExtractorConstant.java | 12 + .../connector/protocol/IoTDBAirGapConnector.java | 21 +- .../queue/ConcurrentIterableLinkedQueue.java | 30 + .../iotdb/commons/pipe/event/EnrichedEvent.java | 45 +- .../commons/pipe/event/PipeSnapshotEvent.java | 4 + .../commons/pipe/event/PipeWritePlanEvent.java | 4 + .../commons/pipe/event/ProgressReportEvent.java | 25 +- .../commons/pipe/extractor/IoTDBExtractor.java | 48 +- .../extractor/IoTDBNonDataRegionExtractor.java | 74 +- .../commons/pipe/receiver/IoTDBFileReceiver.java | 2 +- .../pipe/receiver/PipeReceiverStatusHandler.java | 8 + .../schema/column/ColumnHeaderConstant.java | 7 +- .../table/AlterOrDropTableOperationType.java | 8 +- .../commons/schema/table/InformationSchema.java | 6 + .../apache/iotdb/commons/schema/table/TsTable.java | 3 +- .../schema/table/column/TsTableColumnCategory.java | 2 +- .../commons/schema/view/LogicalViewSchema.java | 13 +- .../multi/FunctionViewExpression.java | 14 + .../viewExpression/unary/LikeViewExpression.java | 84 +- .../apache/iotdb/commons/service/ServiceType.java | 1 + .../iotdb/commons/service/metric/enums/Metric.java | 3 + .../subscription/config/SubscriptionConfig.java | 33 + .../commons/subscription/meta/topic/TopicMeta.java | 3 + .../iotdb/commons/udf/access/RecordIterator.java | 141 +- .../relational/TableBuiltinTableFunction.java | 77 + .../relational/tvf/CapacityTableFunction.java | 119 + .../builtin/relational/tvf/HOPTableFunction.java | 140 + .../relational/tvf/SessionTableFunction.java | 139 + .../relational/tvf/VariationTableFunction.java | 142 + .../udf/builtin/relational/tvf/WindowTVFUtils.java | 55 + .../commons/udf/service/UDFManagementService.java | 5 +- .../org/apache/iotdb/commons/utils/FileUtils.java | 49 + .../apache/iotdb/commons/utils/JVMCommonUtils.java | 2 +- .../org/apache/iotdb/commons/utils/RetryUtils.java | 8 + .../apache/iotdb/commons/utils/StatusUtils.java | 5 + .../iotdb/commons/utils/TimePartitionUtils.java | 7 +- .../iotdb/commons/memory/MemoryBlockTest.java | 67 + .../iotdb/commons/memory/MemoryManagerTest.java | 171 + .../ConcurrentIterableLinkedQueueTest.java | 1 + .../schema/LikeViewExpreesionSerDeTest.java | 48 + .../db/relational/grammar/sql/RelationalSql.g4 | 104 +- .../thrift-ainode/src/main/thrift/ainode.thrift | 16 + .../thrift-commons/src/main/thrift/common.thrift | 2 + .../src/main/thrift/confignode.thrift | 40 +- .../src/main/thrift/pipeconsensus.thrift | 5 +- .../src/main/thrift/datanode.thrift | 19 + pom.xml | 23 +- 1003 files changed, 46046 insertions(+), 18423 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 18aa04de933,c8087e365d3..37cf9b088cd --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@@ -226,57 -216,9 +216,50 @@@ public class TableDistributedPlanGenera @Override public List<PlanNode> visitOffset(OffsetNode node, PlanContext context) { - List<PlanNode> childrenNodes = node.getChild().accept(this, context); - OrderingScheme childOrdering = nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); - if (childOrdering != null) { - nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); - } - - node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); - return Collections.singletonList(node); + return dealWithPlainSingleChildNode(node, context); } + // @Override + public List<PlanNode> visitProject2(ProjectNode node, PlanContext context) { + List<PlanNode> childrenNodes = node.getChild().accept(this, context); + OrderingScheme childOrdering = nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); + boolean containAllSortItem = false; + if (childOrdering != null) { + // the column used for order by has been pruned, we can't copy this node to sub nodeTrees. + containAllSortItem = + ImmutableSet.copyOf(node.getOutputSymbols()).containsAll(childOrdering.getOrderBy()); + } + if (childrenNodes.size() == 1) { + if (containAllSortItem) { + nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); + } + node.setChild(childrenNodes.get(0)); + return Collections.singletonList(node); + } + + boolean containsDiff = + node.getAssignments().getMap().values().stream() + .anyMatch(PushPredicateIntoTableScan::containsDiffFunction); + if (containsDiff) { + if (containAllSortItem) { + nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); + } + node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); + return Collections.singletonList(node); + } + + List<PlanNode> resultNodeList = new ArrayList<>(childrenNodes.size()); + for (PlanNode child : childrenNodes) { + ProjectNode subProjectNode = + new ProjectNode(queryId.genPlanNodeId(), child, node.getAssignments()); + resultNodeList.add(subProjectNode); + if (containAllSortItem) { + nodeOrderingMap.put(subProjectNode.getPlanNodeId(), childOrdering); + } + } + return resultNodeList; + } + @Override public List<PlanNode> visitProject(ProjectNode node, PlanContext context) { List<PlanNode> childrenNodes = node.getChild().accept(this, context);
