This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch force_ci/support_schema_evolution in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3497b6e7b6686ceeabdc107ca44a91d913cc0974 Merge: 5887fa7c90a f71aabf60b6 Author: Tian Jiang <[email protected]> AuthorDate: Tue Jan 13 17:31:53 2026 +0800 Merge branch 'master' into force_ci/support_schema_evolution # Conflicts: # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/FastCompactionPerformerSubTask.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java # pom.xml .github/workflows/cluster-it-1c1d1a.yml | 3 - .github/workflows/vulnerability-check.yml | 42 +- LICENSE | 10 + LICENSE-binary | 4 +- code-coverage/pom.xml | 2 +- dependencies.json | 16 +- distribution/pom.xml | 12 +- example/client-cpp-example/pom.xml | 2 +- example/jdbc/pom.xml | 2 +- example/mqtt-customize/pom.xml | 2 +- example/mqtt/pom.xml | 2 +- example/pipe-count-point-processor/pom.xml | 2 +- example/pipe-opc-ua-sink/pom.xml | 2 +- example/pom.xml | 2 +- example/rest-java-example/pom.xml | 2 +- example/schema/pom.xml | 2 +- example/session/pom.xml | 2 +- .../apache/iotdb/TableModelSessionPoolExample.java | 2 +- example/trigger/pom.xml | 2 +- example/udf/pom.xml | 2 +- integration-test/pom.xml | 33 +- .../it/env/cluster/config/MppCommonConfig.java | 18 + .../it/env/cluster/config/MppDataNodeConfig.java | 6 + .../env/cluster/config/MppSharedCommonConfig.java | 21 + .../iotdb/it/env/cluster/node/AINodeWrapper.java | 4 +- .../iotdb/it/env/cluster/node/DataNodeWrapper.java | 4 + .../it/env/remote/config/RemoteCommonConfig.java | 15 + .../it/env/remote/config/RemoteDataNodeConfig.java | 5 + .../org/apache/iotdb/itbase/env/CommonConfig.java | 6 + .../apache/iotdb/itbase/env/DataNodeConfig.java | 2 + .../iotdb/itbase/runtime/ClusterTestResultSet.java | 16 +- .../iotdb/itbase/runtime/ClusterTestStatement.java | 7 + .../iotdb/ainode/it/AINodeCallInferenceIT.java | 121 + .../iotdb/ainode/it/AINodeClusterConfigIT.java | 3 - .../ainode/it/AINodeConcurrentForecastIT.java | 120 + .../ainode/it/AINodeConcurrentInferenceIT.java | 187 -- ...sterConfigIT.java => AINodeDeviceManageIT.java} | 75 +- .../apache/iotdb/ainode/it/AINodeForecastIT.java | 178 ++ .../iotdb/ainode/it/AINodeInferenceSQLIT.java | 344 --- .../ainode/it/AINodeInstanceManagementIT.java | 79 +- .../iotdb/ainode/it/AINodeModelManageIT.java | 101 +- .../apache/iotdb/ainode/utils/AINodeTestUtils.java | 182 +- .../org/apache/iotdb/db/it/IoTDBFlushQueryIT.java | 82 + .../iotdb/db/it/audit/IoTDBAuditLogBasicIT.java | 2 + .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 16 + .../it/query/IoTDBPredicateConversionTreeIT.java | 293 +++ .../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 11 +- .../db/it/schema/IoTDBAlterTimeSeriesTypeIT.java | 2736 ++++++++++++++++++++ .../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 68 +- .../iotdb/db/it/utils/TSDataTypeTestUtils.java | 113 + .../org/apache/iotdb/db/it/utils/TestUtils.java | 30 +- .../pipe/it/dual/tablemodel/TableModelUtils.java | 8 +- .../manual/basic/IoTDBPipeLifeCycleIT.java | 230 +- .../manual/basic/IoTDBPipePermissionIT.java | 4 +- .../manual/enhanced/IoTDBPipeIdempotentIT.java | 44 +- .../treemodel/auto/basic/IoTDBPipeDataSinkIT.java | 137 +- .../treemodel/auto/basic/IoTDBPipeLifeCycleIT.java | 217 +- .../treemodel/auto/basic/IoTDBPipeProcessorIT.java | 19 +- .../treemodel/auto/basic/IoTDBPipeProtocolIT.java | 87 +- .../auto/basic/IoTDBPipeSinkParallelIT.java | 20 +- .../treemodel/auto/basic/IoTDBPipeSourceIT.java | 460 ++-- .../auto/basic/IoTDBPipeSwitchStatusIT.java | 88 +- .../treemodel/auto/basic/IoTDBPipeSyntaxIT.java | 437 ++-- .../auto/basic/IoTDBTreePatternFormatIT.java | 203 +- .../auto/enhanced/IoTDBPipeAutoConflictIT.java | 126 +- .../auto/enhanced/IoTDBPipeAutoDropIT.java | 42 +- .../auto/enhanced/IoTDBPipeClusterIT.java | 277 +- .../auto/enhanced/IoTDBPipeIdempotentIT.java | 30 +- .../auto/enhanced/IoTDBPipeNullValueIT.java | 22 +- .../auto/enhanced/IoTDBPipeSinkCompressionIT.java | 103 +- .../auto/enhanced/IoTDBPipeWithLoadIT.java | 17 +- .../treemodel/auto/enhanced/PipeNowFunctionIT.java | 153 +- .../treemodel/manual/IoTDBPipeInclusionIT.java | 67 +- .../manual/IoTDBPipeManualConflictIT.java | 102 +- .../manual/IoTDBPipeMetaHistoricalIT.java | 67 +- .../manual/IoTDBPipeMetaLeaderChangeIT.java | 52 +- .../treemodel/manual/IoTDBPipeMetaRestartIT.java | 52 +- .../manual/IoTDBPipeMultiSchemaRegionIT.java | 26 +- .../treemodel/manual/IoTDBPipePermissionIT.java | 204 ++ .../manual/IoTDBPipeTypeConversionISessionIT.java | 10 +- .../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 2 +- .../iotdb/pipe/it/single/IoTDBPipeAggregateIT.java | 14 +- .../iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java | 281 +- .../pipe/it/single/IoTDBPipePermissionIT.java | 30 +- .../iotdb/pipe/it/triple/IoTDBPipeForwardIT.java | 2 + .../relational/it/db/it/IoTDBDeletionTableIT.java | 185 +- .../relational/it/db/it/IoTDBInsertTableIT.java | 6 + .../it/db/it/IoTDBPreparedStatementIT.java | 385 +++ .../it/db/it/IoTDBSetSystemStatusTableIT.java | 1 + .../it/query/object/IoTDBObjectQuery2IT.java | 326 +++ .../it/query/object/IoTDBObjectQueryIT.java | 309 +++ .../it/query/old/IoTDBSimpleQueryTableIT.java | 63 +- .../scalar/IoTDBScalarFunctionTableIT.java | 23 +- .../relational/it/query/recent/IoTDBCteIT.java | 531 +++- .../it/query/recent/IoTDBExceptTableIT.java | 159 ++ .../it/query/recent/IoTDBLengthFunctionIT.java | 119 + .../recent/IoTDBPredicateConversionTableIT.java | 379 +++ .../it/query/recent/IoTDBTableFunctionIT.java | 92 + .../it/query/recent/IoTExplainAnalyzeIT.java | 264 ++ .../informationschema/IoTDBCurrentQueriesIT.java | 263 ++ .../recent/subquery/SubqueryDataSetUtils.java | 4 + .../IoTDBUncorrelatedScalarSubqueryIT.java | 5 + .../view/recent/IoTDBTableViewAddColumnTest.java | 84 + .../it/schema/IoTDBAlterColumnTypeIT.java | 2350 +++++++++++++++++ .../relational/it/schema/IoTDBDatabaseIT.java | 28 +- .../iotdb/relational/it/schema/IoTDBTableIT.java | 164 ++ .../relational/it/session/IoTDBObjectDeleteIT.java | 363 +++ .../it/session/IoTDBObjectDeleteIT2.java | 33 +- .../relational/it/session/IoTDBObjectInsertIT.java | 339 +++ .../it/session/IoTDBObjectInsertIT2.java | 170 ++ .../it/session/IoTDBSessionRelationalIT.java | 29 +- .../iotdb/session/it/IoTDBConnectionsIT.java | 364 +++ .../IoTDBSubscriptionConsumerGroupIT.java | 68 +- .../src/test/resources/ainode-example/config.yaml | 5 - .../{ainode-example/model.pt => object-example.pt} | Bin iotdb-api/external-api/pom.xml | 2 +- iotdb-api/pipe-api/pom.xml | 7 +- .../iotdb/pipe/api/collector/DataCollector.java | 26 +- .../iotdb/pipe/api/collector/TabletCollector.java | 31 +- .../api/customizer/parameter/PipeParameters.java | 31 +- .../event/dml/insertion/TabletInsertionEvent.java | 11 + .../java/org/apache/iotdb/pipe/api/type/Type.java | 5 +- .../customizer/parameter/PipeParametersTest.java | 33 +- iotdb-api/pom.xml | 2 +- iotdb-api/trigger-api/pom.xml | 2 +- iotdb-api/udf-api/pom.xml | 2 +- .../iotdb/udf/api/relational/access/Record.java | 52 +- .../java/org/apache/iotdb/udf/api/type/Type.java | 10 +- iotdb-client/cli/pom.xml | 22 +- .../java/org/apache/iotdb/cli/AbstractCli.java | 5 +- .../org/apache/iotdb/tool/data/ExportData.java | 5 +- iotdb-client/client-cpp/pom.xml | 4 +- iotdb-client/client-cpp/src/main/Common.cpp | 1 + iotdb-client/client-cpp/src/main/Common.h | 3 +- .../client-cpp/src/main/IoTDBRpcDataSet.cpp | 1 + iotdb-client/client-cpp/src/main/Session.cpp | 30 +- iotdb-client/client-cpp/src/main/Session.h | 54 +- .../client-cpp/src/main/SessionDataSet.cpp | 2 + .../src/test/cpp/sessionRelationalIT.cpp | 12 +- iotdb-client/client-py/pom.xml | 8 +- iotdb-client/client-py/session_pool_example.py | 18 +- .../client-py/table_model_session_pool_example.py | 24 +- iotdb-client/isession/pom.xml | 8 +- .../org/apache/iotdb/isession/SessionDataSet.java | 17 + iotdb-client/jdbc/pom.xml | 8 +- .../iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java | 1 + .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 29 +- iotdb-client/pom.xml | 2 +- iotdb-client/service-rpc/pom.xml | 6 +- .../org/apache/iotdb/rpc/IoTDBJDBCDataSet.java | 6 + .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 6 + .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 12 + .../java/org/apache/iotdb/rpc/TSStatusCode.java | 26 +- iotdb-client/session/pom.xml | 24 +- .../org/apache/iotdb/session/NodesSupplier.java | 49 +- .../java/org/apache/iotdb/session/Session.java | 1 + .../payload/SubscriptionSessionDataSet.java | 1 + .../apache/iotdb/session/util/SessionUtils.java | 3 + iotdb-core/ainode/.gitignore | 3 + iotdb-core/ainode/ainode.spec | 138 +- iotdb-core/ainode/build_binary.py | 8 + iotdb-core/ainode/iotdb/ainode/core/config.py | 41 +- iotdb-core/ainode/iotdb/ainode/core/constant.py | 208 +- .../{inference/strategy => device}/__init__.py | 0 .../{model/timerxl => device/backend}/__init__.py | 0 .../timerxl/__init__.py => device/backend/base.py} | 25 + .../__init__.py => device/backend/cpu_backend.py} | 20 + .../__init__.py => device/backend/cuda_backend.py} | 22 + .../__init__.py => device/device_utils.py} | 32 + .../strategy/__init__.py => device/env.py} | 22 + iotdb-core/ainode/iotdb/ainode/core/exception.py | 110 +- .../core/inference/dispatcher/basic_dispatcher.py | 6 +- .../ainode/core/inference/inference_request.py | 37 +- .../core/inference/inference_request_pool.py | 145 +- .../inference/{strategy => pipeline}/__init__.py | 0 .../core/inference/pipeline/basic_pipeline.py | 230 ++ .../core/inference/pipeline/pipeline_loader.py | 58 + .../iotdb/ainode/core/inference/pool_controller.py | 169 +- .../iotdb/ainode/core/inference/pool_group.py | 6 +- .../pool_scheduler/abstract_pool_scheduler.py | 12 +- .../pool_scheduler/basic_pool_scheduler.py | 41 +- .../strategy/abstract_inference_pipeline.py | 60 - .../strategy/timer_sundial_inference_pipeline.py | 51 - .../strategy/timerxl_inference_pipeline.py | 51 - .../ainode/iotdb/ainode/core/inference/utils.py | 45 +- .../iotdb/ainode/core/manager/device_manager.py | 108 + .../iotdb/ainode/core/manager/inference_manager.py | 278 +- .../iotdb/ainode/core/manager/model_manager.py | 157 +- .../ainode/iotdb/ainode/core/manager/utils.py | 17 +- .../ainode/core/model/built_in_model_factory.py | 1238 --------- .../strategy => model/chronos2}/__init__.py | 0 .../iotdb/ainode/core/model/chronos2/base.py | 300 +++ .../ainode/core/model/chronos2/chronos_bolt.py | 703 +++++ .../iotdb/ainode/core/model/chronos2/config.py | 138 + .../iotdb/ainode/core/model/chronos2/dataset.py | 756 ++++++ .../iotdb/ainode/core/model/chronos2/layers.py | 481 ++++ .../iotdb/ainode/core/model/chronos2/model.py | 909 +++++++ .../core/model/chronos2/pipeline_chronos2.py | 476 ++++ .../iotdb/ainode/core/model/chronos2/utils.py | 242 ++ .../__init__.py => model/model_constants.py} | 24 + .../ainode/iotdb/ainode/core/model/model_enums.py | 70 - .../iotdb/ainode/core/model/model_factory.py | 291 --- .../ainode/iotdb/ainode/core/model/model_info.py | 146 +- .../ainode/iotdb/ainode/core/model/model_loader.py | 153 ++ .../iotdb/ainode/core/model/model_storage.py | 775 +++--- .../strategy => model/sktime}/__init__.py | 0 .../ainode/core/model/sktime/arima/config.json | 25 + .../core/model/sktime/configuration_sktime.py | 409 +++ .../model/sktime/exponential_smoothing/config.json | 11 + .../core/model/sktime/gaussian_hmm/config.json | 22 + .../ainode/core/model/sktime/gmm_hmm/config.json | 24 + .../ainode/core/model/sktime/modeling_sktime.py | 180 ++ .../core/model/sktime/naive_forecaster/config.json | 9 + .../ainode/core/model/sktime/pipeline_sktime.py | 115 + .../core/model/sktime/stl_forecaster/config.json | 22 + .../ainode/core/model/sktime/stray/config.json | 11 + .../ainode/core/model/sundial/modeling_sundial.py | 8 +- .../ainode/core/model/sundial/pipeline_sundial.py | 111 + .../strategy => model/timer_xl}/__init__.py | 0 .../{timerxl => timer_xl}/configuration_timer.py | 0 .../model/{timerxl => timer_xl}/modeling_timer.py | 9 +- .../ainode/core/model/timer_xl/pipeline_timer.py | 102 + .../{timerxl => timer_xl}/ts_generation_mixin.py | 0 .../ainode/iotdb/ainode/core/model/uri_utils.py | 137 - iotdb-core/ainode/iotdb/ainode/core/model/utils.py | 137 + iotdb-core/ainode/iotdb/ainode/core/rpc/client.py | 39 - iotdb-core/ainode/iotdb/ainode/core/rpc/handler.py | 135 +- .../ainode/iotdb/ainode/core/util/gpu_mapping.py | 93 - iotdb-core/ainode/iotdb/ainode/core/util/serde.py | 30 +- iotdb-core/ainode/poetry.lock | 2331 ----------------- iotdb-core/ainode/pom.xml | 46 +- iotdb-core/ainode/pyproject.toml | 17 +- .../ainode/resources/conf/iotdb-ainode.properties | 2 +- .../ainode/resources/syncPythonVersion.groovy | 159 -- iotdb-core/antlr/pom.xml | 2 +- .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 2 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 47 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 12 + iotdb-core/confignode/pom.xml | 28 +- .../iotdb/confignode/audit/CNAuditLogger.java | 50 +- .../async/AsyncAINodeHeartbeatClientPool.java | 19 +- .../async/AsyncDataNodeHeartbeatClientPool.java | 1 - .../client/async/CnToDnAsyncRequestType.java | 1 + .../CnToDnInternalServiceAsyncRequestManager.java | 6 + .../rpc/DataNodeAsyncRequestRPCHandler.java | 1 + .../confignode/conf/SystemPropertiesUtils.java | 15 + .../consensus/request/ConfigPhysicalPlan.java | 32 +- .../consensus/request/ConfigPhysicalPlanType.java | 3 + .../request/ConfigPhysicalPlanVisitor.java | 16 + .../request/read/model/GetModelInfoPlan.java | 64 - .../request/read/model/ShowModelPlan.java | 70 - .../request/write/model/CreateModelPlan.java | 79 - .../request/write/model/DropModelPlan.java | 79 - .../request/write/model/UpdateModelInfoPlan.java | 122 - .../pipe/payload/PipeAlterTimeSeriesPlan.java | 110 + .../pipe/payload/PipeDeactivateTemplatePlan.java | 2 +- .../AlterColumnDataTypePlan.java} | 45 +- .../write/template/CreateSchemaTemplatePlan.java | 2 +- .../consensus/response/model/GetModelInfoResp.java | 63 - .../consensus/response/model/ModelTableResp.java | 62 - .../consensus/response/table/DescTableResp.java | 21 +- .../response/template/TemplateInfoResp.java | 2 +- .../response/template/TemplateSetInfoResp.java | 2 +- .../iotdb/confignode/manager/ConfigManager.java | 238 +- .../apache/iotdb/confignode/manager/IManager.java | 46 +- .../iotdb/confignode/manager/ModelManager.java | 245 -- .../confignode/manager/PermissionManager.java | 7 +- .../iotdb/confignode/manager/ProcedureManager.java | 59 +- .../iotdb/confignode/manager/node/NodeManager.java | 1 + .../pipe/agent/task/PipeConfigNodeSubtask.java | 6 +- .../pipe/agent/task/PipeConfigNodeTaskStage.java | 8 +- .../pipe/coordinator/task/PipeTaskCoordinator.java | 16 +- .../coordinator/task/PipeTaskCoordinatorLock.java | 52 +- .../receiver/protocol/IoTDBConfigNodeReceiver.java | 513 ++-- .../PipeConfigPhysicalPlanTSStatusVisitor.java | 18 + .../sink/protocol/IoTDBConfigRegionAirGapSink.java | 6 +- .../pipe/sink/protocol/IoTDBConfigRegionSink.java | 6 +- .../pipe/source/ConfigRegionListeningFilter.java | 8 +- .../pipe/source/IoTDBConfigRegionSource.java | 72 +- ...nfigPhysicalPlanTablePrivilegeParseVisitor.java | 389 --- ...ava => PipeConfigTablePatternParseVisitor.java} | 9 +- .../PipeConfigTablePrivilegeParseVisitor.java | 378 +++ ....java => PipeConfigTableScopeParseVisitor.java} | 2 +- ...java => PipeConfigTreePatternParseVisitor.java} | 29 +- .../PipeConfigTreePrivilegeParseVisitor.java | 610 +++++ ...r.java => PipeConfigTreeScopeParseVisitor.java} | 2 +- .../manager/schema/ClusterSchemaManager.java | 58 +- .../iotdb/confignode/persistence/ModelInfo.java | 378 --- .../confignode/persistence/auth/AuthorInfo.java | 7 + .../persistence/auth/AuthorPlanExecutor.java | 20 + .../persistence/auth/IAuthorPlanExecutor.java | 5 + .../persistence/executor/ConfigPlanExecutor.java | 30 +- .../persistence/pipe/PipePluginInfo.java | 103 +- .../schema/CNPhysicalPlanGenerator.java | 2 +- .../persistence/schema/ClusterSchemaInfo.java | 70 +- .../confignode/persistence/schema/ConfigMTree.java | 97 +- .../persistence/schema/ConfigMTreeStore.java | 2 +- .../persistence/schema/TemplateTable.java | 2 +- .../schema/mnode/impl/ConfigTableNode.java | 15 + .../schema/mnode/info/ConfigTableInfo.java | 16 + .../procedure/impl/model/CreateModelProcedure.java | 250 -- .../procedure/impl/model/DropModelProcedure.java | 200 -- .../procedure/impl/node/RemoveAINodeProcedure.java | 17 +- .../impl/pipe/task/AlterPipeProcedureV2.java | 1 + .../schema/AlterEncodingCompressorProcedure.java | 8 +- .../impl/schema/AlterLogicalViewProcedure.java | 10 +- .../schema/AlterTimeSeriesDataTypeProcedure.java | 413 +++ .../impl/schema/DataNodeRegionTaskExecutor.java | 17 +- .../impl/schema/DataNodeTSStatusTaskExecutor.java | 39 + .../impl/schema/DeactivateTemplateProcedure.java | 34 +- .../impl/schema/DeleteLogicalViewProcedure.java | 32 +- .../impl/schema/DeleteTimeSeriesProcedure.java | 25 +- .../procedure/impl/schema/SchemaUtils.java | 14 +- .../impl/schema/SetTemplateProcedure.java | 14 +- .../impl/schema/UnsetTemplateProcedure.java | 2 +- .../table/AlterTableColumnDataTypeProcedure.java | 212 ++ .../impl/schema/table/DeleteDevicesProcedure.java | 96 +- .../schema/table/DropTableColumnProcedure.java | 2 +- .../procedure/state/RemoveAINodeState.java | 1 - .../schema/AlterTableColumnDataTypeState.java} | 13 +- .../AlterTimeSeriesDataTypeState.java} | 10 +- .../procedure/store/ProcedureFactory.java | 30 +- .../confignode/procedure/store/ProcedureType.java | 7 + .../thrift/ConfigNodeRPCServiceProcessor.java | 31 +- .../request/ConfigPhysicalPlanSerDeTest.java | 35 +- ...t.java => PipeConfigScopeParseVisitorTest.java} | 2 +- ...=> PipeConfigTablePatternParseVisitorTest.java} | 12 +- ... => PipeConfigTreePatternParseVisitorTest.java} | 25 +- .../PipeConfigTreePrivilegeParseVisitorTest.java | 316 +++ .../persistence/CNPhysicalPlanGeneratorTest.java | 2 +- .../iotdb/confignode/persistence/PipeInfoTest.java | 16 +- .../persistence/schema/ClusterSchemaInfoTest.java | 2 +- .../persistence/schema/ConfigMTreeTest.java | 2 +- .../persistence/schema/TemplateTableTest.java | 2 +- .../pipe/receiver/PipeEnrichedProcedureTest.java | 2 +- .../schema/DataNodeRegionTaskExecutorTest.java | 81 + .../schema/DeactivateTemplateProcedureTest.java | 2 +- .../impl/schema/UnsetTemplateProcedureTest.java | 2 +- iotdb-core/consensus/pom.xml | 28 +- iotdb-core/datanode/pom.xml | 56 +- .../org/apache/iotdb/db/audit/DNAuditLogger.java | 398 +-- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 11 +- .../iotdb/db/auth/ClusterAuthorityFetcher.java | 59 +- .../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 5 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 110 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 89 +- .../db/consensus/DataRegionConsensusImpl.java | 6 + .../dataregion/DataExecutionVisitor.java | 13 + .../dataregion/DataRegionStateMachine.java | 15 +- .../schemaregion/SchemaExecutionVisitor.java | 102 +- .../ChunkTypeInconsistentException.java} | 39 +- .../DataTypeInconsistentException.java} | 16 +- .../db/exception/ainode/GetModelInfoException.java | 2 +- .../db/pipe/agent/runtime/PipeAgentLauncher.java | 3 +- .../agent/task/connection/PipeEventCollector.java | 22 +- .../subtask/processor/PipeProcessorSubtask.java | 29 +- .../agent/task/subtask/sink/PipeSinkSubtask.java | 11 +- .../db/pipe/event/common/PipeInsertionEvent.java | 1 + .../db/pipe/event/common/row/PipeRowCollector.java | 35 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 150 +- .../common/tablet/PipeRawTabletEventConverter.java | 72 + .../common/tablet/PipeRawTabletInsertionEvent.java | 34 +- .../event/common/tablet/PipeTabletCollector.java | 61 + .../tablet/parser/TabletInsertionEventParser.java | 7 +- .../TabletInsertionEventTablePatternParser.java | 18 +- .../TabletInsertionEventTreePatternParser.java | 49 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 126 +- .../tsfile/parser/TsFileInsertionEventParser.java | 8 +- .../parser/TsFileInsertionEventParserProvider.java | 26 +- .../query/TsFileInsertionEventQueryParser.java | 48 +- .../scan/TsFileInsertionEventScanParser.java | 60 +- .../table/TsFileInsertionEventTableParser.java | 38 +- .../processor/aggregate/AggregateProcessor.java | 3 + .../protocol/thrift/IoTDBDataNodeReceiver.java | 11 +- .../transform/converter/ValueConverter.java | 6 +- .../visitor/PipePlanToStatementVisitor.java | 1 + .../visitor/PipeStatementExceptionVisitor.java | 8 +- .../visitor/PipeStatementTSStatusVisitor.java | 3 +- ...leStatementDataTypeConvertExecutionVisitor.java | 2 +- .../resource/memory/InsertNodeMemoryEstimator.java | 1 + .../evolvable/batch/PipeTabletEventBatch.java | 6 + .../batch/PipeTabletEventTsFileBatch.java | 6 +- .../request/PipeTransferTabletBatchReqV2.java | 7 +- .../request/PipeTransferTabletRawReq.java | 110 +- .../request/PipeTransferTabletRawReqV2.java | 50 +- .../airgap/IoTDBSchemaRegionAirGapSink.java | 6 +- .../sink/protocol/legacy/IoTDBLegacyPipeSink.java | 7 +- .../sink/protocol/opcda/OpcDaServerHandle.java | 2 + .../db/pipe/sink/protocol/opcua/OpcUaSink.java | 302 ++- .../sink/protocol/opcua/client/ClientRunner.java | 112 + .../opcua/client/IoTDBKeyStoreLoaderClient.java | 127 + .../protocol/opcua/client/IoTDBOpcUaClient.java | 301 +++ .../opcua/{ => server}/OpcUaKeyStoreLoader.java | 2 +- .../opcua/{ => server}/OpcUaNameSpace.java | 171 +- .../opcua/{ => server}/OpcUaServerBuilder.java | 97 +- .../handler/PipeConsensusDeleteEventHandler.java | 2 +- .../handler/PipeTransferTrackableHandler.java | 16 +- .../async/handler/PipeTransferTsFileHandler.java | 15 +- .../thrift/sync/IoTDBDataRegionSyncSink.java | 3 +- .../thrift/sync/IoTDBSchemaRegionSink.java | 10 +- .../sink/protocol/writeback/WriteBackSink.java | 70 +- .../pipe/sink/util/TabletStatementConverter.java | 476 ++++ .../sink/util/sorter/InsertEventDataAdapter.java | 127 + .../util/sorter/InsertTabletStatementAdapter.java | 118 + ...EventSorter.java => PipeInsertEventSorter.java} | 100 +- .../sorter/PipeTableModelTabletEventSorter.java | 67 +- .../sorter/PipeTreeModelTabletEventSorter.java | 48 +- .../db/pipe/sink/util/sorter/TabletAdapter.java | 113 + .../matcher/CachedSchemaPatternMatcher.java | 8 +- .../schemaregion/IoTDBSchemaRegionSource.java | 80 +- .../PipePlanTreePrivilegeParseVisitor.java | 355 +++ ...r.java => PipeTableStatementToPlanVisitor.java} | 2 +- .../PipeTreeStatementToPlanVisitor.java | 60 + .../db/protocol/client/AINodeClientFactory.java | 133 - .../iotdb/db/protocol/client/ConfigNodeClient.java | 37 +- .../protocol/client/DataNodeClientPoolFactory.java | 42 +- .../db/protocol/client/ainode/AINodeClient.java | 401 --- .../client/ainode/AINodeClientManager.java | 75 - .../iotdb/db/protocol/client/an/AINodeClient.java | 321 +++ .../client/an/AINodeClientManager.java} | 38 +- .../protocol/rest/v2/impl/RestApiServiceImpl.java | 16 +- .../iotdb/db/protocol/session/ClientSession.java | 23 + .../iotdb/db/protocol/session/IClientSession.java | 48 + .../db/protocol/session/InternalClientSession.java | 24 + .../db/protocol/session/MqttClientSession.java | 23 + .../db/protocol/session/PreparedStatementInfo.java | 99 + .../db/protocol/session/RestClientSession.java | 25 + .../iotdb/db/protocol/session/SessionManager.java | 66 +- .../thrift/handler/BaseServerContextHandler.java | 1 + .../protocol/thrift/impl/ClientRPCServiceImpl.java | 338 ++- .../impl/DataNodeInternalRPCServiceImpl.java | 38 + .../db/queryengine/common/ConnectionInfo.java | 64 + .../db/queryengine/common/MPPQueryContext.java | 128 +- .../iotdb/db/queryengine/common/QueryId.java | 7 + .../common/header/DatasetHeaderFactory.java | 4 + .../common/schematree/ClusterSchemaTree.java | 2 +- .../queryengine/common/schematree/ISchemaTree.java | 2 +- .../schematree/visitor/SchemaTreeVisitor.java | 2 +- .../SchemaTreeVisitorWithLimitOffsetWrapper.java | 2 +- .../execution/aggregation/AccumulatorFactory.java | 1 + .../execution/aggregation/AvgAccumulator.java | 1 + .../execution/aggregation/ExtremeAccumulator.java | 22 +- .../aggregation/FirstValueAccumulator.java | 37 +- .../aggregation/LastValueAccumulator.java | 37 +- .../aggregation/MaxMinByBaseAccumulator.java | 6 + .../execution/aggregation/MaxValueAccumulator.java | 14 +- .../execution/aggregation/MinValueAccumulator.java | 14 +- .../execution/aggregation/SumAccumulator.java | 1 + .../execution/aggregation/VarianceAccumulator.java | 1 + .../execution/exchange/SharedTsBlockQueue.java | 4 + .../exchange/source/LocalSourceHandle.java | 5 + .../execution/executor/RegionWriteExecutor.java | 27 +- .../fragment/FragmentInstanceContext.java | 41 +- .../fragment/FragmentInstanceExecution.java | 35 +- .../execution/fragment/QueryStatistics.java | 10 +- .../execution/memory/LocalMemoryManager.java | 1 - .../execution/operator/AggregationUtil.java | 1 + .../execution/operator/ExplainAnalyzeOperator.java | 76 +- .../operator/process/AbstractIntoOperator.java | 6 +- .../operator/process/AbstractTreeIntoOperator.java | 8 +- .../operator/process/DeviceViewIntoOperator.java | 110 +- .../process/InsertTabletStatementGenerator.java | 4 + .../operator/process/TableIntoOperator.java | 5 + .../execution/operator/process/TopKOperator.java | 14 +- .../operator/process/TransformOperator.java | 1 + .../operator/process/TreeIntoOperator.java | 37 +- .../operator/process/ai/InferenceOperator.java | 82 +- .../operator/process/function/partition/Slice.java | 62 +- .../process/join/merge/MergeSortComparator.java | 1 + .../expression/PatternExpressionComputation.java | 5 +- .../process/window/TableWindowOperator.java | 2 + .../process/window/function/value/LagFunction.java | 1 + .../window/function/value/LeadFunction.java | 1 + .../process/window/utils/RowComparator.java | 3 + .../operator/schema/SchemaFetchScanOperator.java | 2 +- .../schema/source/SchemaSourceFactory.java | 2 +- .../schema/source/TimeSeriesSchemaSource.java | 2 +- .../execution/operator/source/FileLoaderUtils.java | 73 +- .../execution/operator/source/SeriesScanUtil.java | 715 ++++- .../source/relational/CteScanOperator.java | 119 + .../InformationSchemaContentSupplierFactory.java | 331 ++- .../relational/LastQueryAggTableScanOperator.java | 1 + .../relational/aggregation/AccumulatorFactory.java | 2 + .../ApproxCountDistinctAccumulator.java | 1 + .../relational/aggregation/AvgAccumulator.java | 2 + .../relational/aggregation/ExtremeAccumulator.java | 21 +- .../relational/aggregation/FirstAccumulator.java | 36 +- .../relational/aggregation/FirstByAccumulator.java | 22 +- .../relational/aggregation/LastAccumulator.java | 36 +- .../relational/aggregation/LastByAccumulator.java | 12 +- .../aggregation/MaskedRecordIterator.java | 2 - .../relational/aggregation/MaxAccumulator.java | 22 +- .../relational/aggregation/MinAccumulator.java | 22 +- .../relational/aggregation}/RecordIterator.java | 70 +- .../relational/aggregation/SumAccumulator.java | 2 + .../aggregation/TableMaxMinByBaseAccumulator.java | 5 + .../aggregation/TableVarianceAccumulator.java | 2 + .../UserDefinedAggregateFunctionAccumulator.java | 1 - .../source/relational/aggregation/Utils.java | 2 + .../GroupedApproxCountDistinctAccumulator.java | 1 + .../aggregation/grouped/GroupedAvgAccumulator.java | 1 + .../grouped/GroupedExtremeAccumulator.java | 8 + .../grouped/GroupedFirstAccumulator.java | 8 + .../grouped/GroupedFirstByAccumulator.java | 9 + .../grouped/GroupedLastAccumulator.java | 8 + .../grouped/GroupedLastByAccumulator.java | 9 + .../aggregation/grouped/GroupedMaxAccumulator.java | 8 + .../grouped/GroupedMaxMinByBaseAccumulator.java | 15 + .../aggregation/grouped/GroupedMinAccumulator.java | 8 + .../grouped/GroupedModeAccumulator.java | 4 + .../aggregation/grouped/GroupedSumAccumulator.java | 1 + .../GroupedUserDefinedAggregateAccumulator.java | 2 +- .../grouped/GroupedVarianceAccumulator.java | 1 + .../operator/window/WindowManagerFactory.java | 2 + .../relational/ColumnTransformerBuilder.java | 57 +- .../iotdb/db/queryengine/plan/Coordinator.java | 491 +++- .../db/queryengine/plan/analyze/Analysis.java | 2 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 161 +- .../plan/analyze/ClusterPartitionFetcher.java | 4 + .../db/queryengine/plan/analyze/IModelFetcher.java | 4 - .../db/queryengine/plan/analyze/ModelFetcher.java | 51 +- .../queryengine/plan/analyze/SelectIntoUtils.java | 42 +- .../plan/analyze/TemplatedAggregationAnalyze.java | 2 +- .../queryengine/plan/analyze/TemplatedAnalyze.java | 2 +- .../dualkeycache/impl/CacheEntryGroupImpl.java | 6 + .../schema/dualkeycache/impl/DualKeyCacheImpl.java | 61 +- .../schema/dualkeycache/impl/ICacheEntryGroup.java | 3 + .../analyze/schema/AutoCreateSchemaExecutor.java | 2 +- .../analyze/schema/ClusterSchemaFetchExecutor.java | 2 +- .../plan/analyze/schema/ClusterSchemaFetcher.java | 2 +- .../plan/analyze/schema/ISchemaFetcher.java | 2 +- .../plan/analyze/schema/TemplateSchemaFetcher.java | 2 +- .../plan/execution/IQueryExecution.java | 2 + .../queryengine/plan/execution/QueryExecution.java | 5 + .../plan/execution/config/ConfigExecution.java | 5 + .../execution/config/TableConfigTaskVisitor.java | 123 +- .../execution/config/TreeConfigTaskVisitor.java | 51 +- .../config/executor/ClusterConfigTaskExecutor.java | 276 +- .../config/executor/IConfigTaskExecutor.java | 19 +- .../config/metadata/AlterTimeSeriesTask.java | 57 + ...DevicesTask.java => ShowAvailableUrlsTask.java} | 32 +- ...eateTrainingTask.java => CreateTuningTask.java} | 8 +- .../config/metadata/ai/ShowAIDevicesTask.java | 15 +- .../relational/AlterColumnDataTypeTask.java | 54 + .../relational/DescribeTableDetailsTask.java | 14 +- .../template/ShowNodesInSchemaTemplateTask.java | 2 +- .../metadata/template/ShowSchemaTemplateTask.java | 2 +- .../execution/config/session/DeallocateTask.java | 72 + .../plan/execution/config/session/PrepareTask.java | 84 + .../session/PreparedStatementMemoryManager.java | 157 ++ .../execution/config/sys/pipe/AlterPipeTask.java | 10 +- .../execution/config/sys/pipe/CreatePipeTask.java | 8 +- .../execution/config/sys/pipe/ShowPipeTask.java | 5 +- .../TableModelStatementMemorySourceVisitor.java | 34 +- .../predicate/ConvertPredicateToFilterVisitor.java | 134 +- .../db/queryengine/plan/parser/ASTVisitor.java | 154 +- .../plan/planner/LocalExecutionPlanner.java | 8 + .../plan/planner/LogicalPlanBuilder.java | 2 +- .../plan/planner/LogicalPlanVisitor.java | 21 +- .../plan/planner/OperatorTreeGenerator.java | 2 + .../plan/planner/TableOperatorGenerator.java | 79 +- .../planner/distribution/ExchangeNodeAdder.java | 6 +- .../plan/planner/distribution/SourceRewriter.java | 7 +- .../memory/FakedMemoryReservationManager.java | 11 + .../planner/memory/MemoryReservationManager.java | 29 + .../NotThreadSafeMemoryReservationManager.java | 23 + .../memory/ThreadSafeMemoryReservationManager.java | 13 + .../plan/planner/plan/DistributedQueryPlan.java | 11 + .../plan/planner/plan/node/PlanGraphPrinter.java | 30 + .../plan/planner/plan/node/PlanNodeType.java | 22 +- .../plan/planner/plan/node/PlanVisitor.java | 19 + .../metadata/read/LevelTimeSeriesCountNode.java | 2 +- .../metadata/read/SeriesSchemaFetchScanNode.java | 2 +- .../node/metadata/read/TimeSeriesCountNode.java | 2 +- .../metadata/read/TimeSeriesSchemaScanNode.java | 2 +- .../node/metadata/write/AlterTimeSeriesNode.java | 87 +- .../plan/node/pipe/PipeEnrichedInsertNode.java | 7 + .../plan/node/process/AI/InferenceNode.java | 3 +- .../plan/node/source/LastQueryScanNode.java | 4 + .../planner/plan/node/source/ShowQueriesNode.java | 2 +- .../plan/node/write/InsertMultiTabletsNode.java | 9 + .../plan/planner/plan/node/write/InsertNode.java | 5 + .../planner/plan/node/write/InsertRowNode.java | 26 +- .../planner/plan/node/write/InsertRowsNode.java | 9 + .../plan/node/write/InsertRowsOfOneDeviceNode.java | 9 + .../planner/plan/node/write/InsertTabletNode.java | 40 +- .../plan/planner/plan/node/write/ObjectNode.java | 331 +++ .../plan/node/write/RelationalInsertRowNode.java | 11 + .../plan/node/write/RelationalInsertRowsNode.java | 57 +- .../node/write/RelationalInsertTabletNode.java | 147 ++ .../parameter/DeviceViewIntoPathDescriptor.java | 5 +- .../planner/plan/parameter/IntoPathDescriptor.java | 5 +- .../parameter/model/ModelInferenceDescriptor.java | 61 +- .../plan/relational/analyzer/Analysis.java | 31 + .../plan/relational/analyzer/Analyzer.java | 6 + .../relational/analyzer/ExpressionAnalyzer.java | 8 +- .../plan/relational/analyzer/Scope.java | 36 +- .../relational/analyzer/StatementAnalyzer.java | 176 +- .../analyzer/StatementAnalyzerFactory.java | 21 +- .../predicate/ConvertPredicateToFilterVisitor.java | 116 +- .../function/TableBuiltinTableFunction.java | 6 +- .../function/tvf/ClassifyTableFunction.java | 383 +++ .../function/tvf/ForecastTableFunction.java | 358 +-- .../plan/relational/metadata/Metadata.java | 16 +- .../relational/metadata/TableMetadataImpl.java | 70 +- .../metadata/cache/DeviceSchemaRequestCache.java | 8 +- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 4 +- .../fetcher/TableDeviceSchemaValidator.java | 27 +- .../fetcher/TableHeaderSchemaValidator.java | 389 +++ .../fetcher/cache/TableDeviceCacheEntry.java | 8 +- .../fetcher/cache/TableDeviceLastCache.java | 56 +- .../fetcher/cache/TableDeviceSchemaCache.java | 6 +- .../fetcher/cache/TreeDeviceNormalSchema.java | 6 +- .../cache/TreeDeviceSchemaCacheManager.java | 2 +- .../plan/relational/planner/CteMaterializer.java | 355 +++ .../plan/relational/planner/QueryPlanner.java | 50 +- .../plan/relational/planner/RelationPlanner.java | 159 +- .../plan/relational/planner/SubqueryPlanner.java | 27 +- .../relational/planner/TableLogicalPlanner.java | 28 +- .../plan/relational/planner/TableModelPlanner.java | 25 +- .../planner/distribute/AddExchangeNodes.java | 50 +- .../distribute/TableDistributedPlanGenerator.java | 8 +- .../distribute/TableDistributedPlanner.java | 17 +- ...ithUncorrelatedScalarSubqueryReconstructor.java | 245 ++ .../iterative/rule/EvaluateEmptyIntersect.java | 63 + ...istinctAsUnion.java => ImplementExceptAll.java} | 78 +- ...on.java => ImplementExceptDistinctAsUnion.java} | 52 +- .../rule/ImplementIntersectDistinctAsUnion.java | 3 +- .../planner/iterative/rule/MergeExcept.java | 47 + .../planner/iterative/rule/MergeIntersect.java | 47 + .../iterative/rule/PruneDistinctAggregation.java | 27 +- .../iterative/rule/PruneExceptSourceColumns.java | 54 + .../rule/PruneIntersectSourceColumns.java | 55 + .../iterative/rule/RemoveEmptyExceptBranches.java | 139 + .../planner/iterative/rule/SetOperationMerge.java | 123 +- .../iterative/rule/SetOperationNodeTranslator.java | 4 +- .../plan/relational/planner/node/CteScanNode.java | 132 + .../plan/relational/planner/node/ExceptNode.java | 99 + .../plan/relational/planner/node/Patterns.java | 20 +- .../DataNodeLocationSupplierFactory.java | 4 +- .../optimizations/LogicalOptimizeFactory.java | 43 +- .../PushLimitOffsetIntoTableScan.java | 7 + .../optimizations/PushPredicateIntoTableScan.java | 100 + .../optimizations/TransformSortToStreamSort.java | 7 + .../optimizations/UnaliasSymbolReferences.java | 38 + .../plan/relational/security/AccessControl.java | 10 + .../relational/security/AccessControlImpl.java | 143 +- .../relational/security/AllowAllAccessControl.java | 17 + .../relational/security/ITableAuthCheckerImpl.java | 68 +- .../security/TreeAccessCheckVisitor.java | 333 ++- .../plan/relational/sql/ParameterExtractor.java | 121 + .../sql/ast/AbstractQueryDeviceWithCache.java | 21 + .../relational/sql/ast/AbstractTraverseDevice.java | 48 + .../plan/relational/sql/ast/AddColumn.java | 13 + .../plan/relational/sql/ast/AliasedRelation.java | 14 + .../plan/relational/sql/ast/AllColumns.java | 13 + .../plan/relational/sql/ast/AllRows.java | 9 + .../relational/sql/ast/AlterColumnDataType.java | 133 + .../plan/relational/sql/ast/AlterDB.java | 9 + .../plan/relational/sql/ast/AlterPipe.java | 15 + .../plan/relational/sql/ast/AnchorPattern.java | 11 + .../sql/ast/ArithmeticBinaryExpression.java | 12 + .../sql/ast/ArithmeticUnaryExpression.java | 11 + .../plan/relational/sql/ast/AsofJoinOn.java | 11 + .../sql/ast/AstMemoryEstimationHelper.java | 123 + .../plan/relational/sql/ast/AstVisitor.java | 24 + .../plan/relational/sql/ast/BetweenPredicate.java | 13 + .../plan/relational/sql/ast/BinaryLiteral.java | 11 + .../plan/relational/sql/ast/BooleanLiteral.java | 10 + .../queryengine/plan/relational/sql/ast/Cast.java | 12 + .../plan/relational/sql/ast/ClearCache.java | 11 + .../relational/sql/ast/CoalesceExpression.java | 11 + .../plan/relational/sql/ast/ColumnDefinition.java | 15 + .../plan/relational/sql/ast/Columns.java | 11 + .../relational/sql/ast/ComparisonExpression.java | 34 +- .../plan/relational/sql/ast/CountDevice.java | 8 + .../plan/relational/sql/ast/CountStatement.java | 13 + .../plan/relational/sql/ast/CreateDB.java | 9 + .../plan/relational/sql/ast/CreateFunction.java | 13 + .../plan/relational/sql/ast/CreateIndex.java | 13 + .../plan/relational/sql/ast/CreateModel.java | 13 + .../relational/sql/ast/CreateOrUpdateDevice.java | 16 + .../plan/relational/sql/ast/CreatePipe.java | 52 +- .../plan/relational/sql/ast/CreatePipePlugin.java | 14 + .../plan/relational/sql/ast/CreateTable.java | 26 + .../plan/relational/sql/ast/CreateTopic.java | 13 + .../plan/relational/sql/ast/CreateTraining.java | 15 + .../plan/relational/sql/ast/CreateView.java | 14 + .../plan/relational/sql/ast/CurrentDatabase.java | 11 + .../plan/relational/sql/ast/CurrentTime.java | 14 + .../plan/relational/sql/ast/CurrentUser.java | 11 + .../plan/relational/sql/ast/DatabaseStatement.java | 9 + .../sql/ast/{Table.java => Deallocate.java} | 56 +- .../plan/relational/sql/ast/DecimalLiteral.java | 11 + .../plan/relational/sql/ast/Delete.java | 12 + .../plan/relational/sql/ast/DeleteDevice.java | 19 +- .../relational/sql/ast/DereferenceExpression.java | 13 + .../plan/relational/sql/ast/DescribeTable.java | 11 + .../plan/relational/sql/ast/DoubleLiteral.java | 10 + .../plan/relational/sql/ast/DropColumn.java | 12 + .../plan/relational/sql/ast/DropDB.java | 11 + .../plan/relational/sql/ast/DropFunction.java | 11 + .../plan/relational/sql/ast/DropIndex.java | 12 + .../plan/relational/sql/ast/DropModel.java | 12 + .../plan/relational/sql/ast/DropPipe.java | 11 + .../plan/relational/sql/ast/DropPipePlugin.java | 12 + .../plan/relational/sql/ast/DropSubscription.java | 12 + .../plan/relational/sql/ast/DropTable.java | 12 + .../plan/relational/sql/ast/DropTopic.java | 12 + .../plan/relational/sql/ast/EmptyPattern.java | 11 + .../plan/relational/sql/ast/Except.java | 12 + .../plan/relational/sql/ast/ExcludedPattern.java | 12 + .../plan/relational/sql/ast/Execute.java | 108 + .../plan/relational/sql/ast/ExecuteImmediate.java | 112 + .../plan/relational/sql/ast/ExistsPredicate.java | 12 + .../plan/relational/sql/ast/Explain.java | 11 + .../plan/relational/sql/ast/ExplainAnalyze.java | 12 + .../plan/relational/sql/ast/ExtendRegion.java | 11 + .../plan/relational/sql/ast/Extract.java | 11 + .../plan/relational/sql/ast/FetchDevice.java | 19 + .../plan/relational/sql/ast/FieldReference.java | 11 + .../queryengine/plan/relational/sql/ast/Fill.java | 16 + .../plan/relational/sql/ast/FrameBound.java | 13 + .../plan/relational/sql/ast/FunctionCall.java | 23 + .../plan/relational/sql/ast/GenericDataType.java | 12 + .../plan/relational/sql/ast/GenericLiteral.java | 12 + .../plan/relational/sql/ast/GroupBy.java | 11 + .../plan/relational/sql/ast/GroupingSets.java | 14 + .../plan/relational/sql/ast/Identifier.java | 11 + .../plan/relational/sql/ast/IfExpression.java | 14 + .../plan/relational/sql/ast/InListExpression.java | 12 + .../plan/relational/sql/ast/InPredicate.java | 12 + .../plan/relational/sql/ast/Insert.java | 13 + .../plan/relational/sql/ast/InsertRow.java | 11 +- .../plan/relational/sql/ast/InsertRows.java | 33 +- .../plan/relational/sql/ast/InsertTablet.java | 9 + .../plan/relational/sql/ast/Intersect.java | 12 + .../relational/sql/ast/IsNotNullPredicate.java | 11 + .../plan/relational/sql/ast/IsNullPredicate.java | 11 + .../queryengine/plan/relational/sql/ast/Join.java | 13 + .../plan/relational/sql/ast/JoinCriteria.java | 4 +- .../plan/relational/sql/ast/JoinOn.java | 13 + .../plan/relational/sql/ast/JoinUsing.java | 11 + .../plan/relational/sql/ast/KillQuery.java | 12 + .../plan/relational/sql/ast/LikePredicate.java | 13 + .../queryengine/plan/relational/sql/ast/Limit.java | 11 + .../plan/relational/sql/ast/LoadModel.java | 14 + .../plan/relational/sql/ast/LoadTsFile.java | 97 +- .../plan/relational/sql/ast/LogicalExpression.java | 11 + .../plan/relational/sql/ast/LongLiteral.java | 13 +- .../plan/relational/sql/ast/MeasureDefinition.java | 13 + .../plan/relational/sql/ast/MigrateRegion.java | 10 + .../plan/relational/sql/ast/NaturalJoin.java | 9 + .../queryengine/plan/relational/sql/ast/Node.java | 9 +- .../plan/relational/sql/ast/NotExpression.java | 11 + .../plan/relational/sql/ast/NullIfExpression.java | 13 + .../plan/relational/sql/ast/NullLiteral.java | 11 + .../plan/relational/sql/ast/NumericParameter.java | 11 + .../plan/relational/sql/ast/Offset.java | 11 + .../relational/sql/ast/OneOrMoreQuantifier.java | 12 + .../plan/relational/sql/ast/OrderBy.java | 11 + .../plan/relational/sql/ast/Parameter.java | 11 + .../relational/sql/ast/PatternAlternation.java | 12 + .../relational/sql/ast/PatternConcatenation.java | 12 + .../relational/sql/ast/PatternPermutation.java | 12 + .../sql/ast/PatternRecognitionRelation.java | 22 + .../plan/relational/sql/ast/PatternVariable.java | 12 + .../plan/relational/sql/ast/PipeEnriched.java | 13 + .../plan/relational/sql/ast/Prepare.java | 99 + .../plan/relational/sql/ast/ProcessingMode.java | 11 + .../plan/relational/sql/ast/Property.java | 12 + .../plan/relational/sql/ast/QualifiedName.java | 20 +- .../sql/ast/QuantifiedComparisonExpression.java | 13 + .../plan/relational/sql/ast/QuantifiedPattern.java | 13 + .../queryengine/plan/relational/sql/ast/Query.java | 53 + .../relational/sql/ast/QuerySpecification.java | 22 + .../plan/relational/sql/ast/RangeQuantifier.java | 14 + .../plan/relational/sql/ast/ReconstructRegion.java | 11 + .../sql/ast/RelationalAuthorStatement.java | 19 + .../plan/relational/sql/ast/RemoveAINode.java | 10 + .../plan/relational/sql/ast/RemoveConfigNode.java | 10 + .../plan/relational/sql/ast/RemoveDataNode.java | 11 + .../plan/relational/sql/ast/RemoveRegion.java | 11 + .../plan/relational/sql/ast/RenameColumn.java | 19 +- .../plan/relational/sql/ast/RenameTable.java | 18 +- .../queryengine/plan/relational/sql/ast/Row.java | 11 + .../relational/sql/ast/SearchedCaseExpression.java | 13 + .../plan/relational/sql/ast/Select.java | 10 + .../plan/relational/sql/ast/SetColumnComment.java | 13 + .../plan/relational/sql/ast/SetProperties.java | 12 + .../plan/relational/sql/ast/SetSqlDialect.java | 10 + .../plan/relational/sql/ast/SetTableComment.java | 12 + .../plan/relational/sql/ast/ShowAIDevices.java | 12 + .../plan/relational/sql/ast/ShowAINodes.java | 11 + ...ShowConfigNodes.java => ShowAvailableUrls.java} | 22 +- .../plan/relational/sql/ast/ShowCluster.java | 11 + .../plan/relational/sql/ast/ShowClusterId.java | 11 + .../plan/relational/sql/ast/ShowConfigNodes.java | 11 + .../plan/relational/sql/ast/ShowConfiguration.java | 1 - .../relational/sql/ast/ShowCurrentDatabase.java | 11 + .../relational/sql/ast/ShowCurrentSqlDialect.java | 11 + .../relational/sql/ast/ShowCurrentTimestamp.java | 11 + .../plan/relational/sql/ast/ShowCurrentUser.java | 11 + .../plan/relational/sql/ast/ShowDB.java | 10 + .../plan/relational/sql/ast/ShowDataNodes.java | 11 + .../plan/relational/sql/ast/ShowDevice.java | 12 + .../plan/relational/sql/ast/ShowFunctions.java | 11 + .../plan/relational/sql/ast/ShowIndex.java | 12 + .../plan/relational/sql/ast/ShowLoadedModels.java | 13 + .../plan/relational/sql/ast/ShowModels.java | 12 + .../plan/relational/sql/ast/ShowPipePlugins.java | 12 + .../plan/relational/sql/ast/ShowPipes.java | 13 + .../plan/relational/sql/ast/ShowRegions.java | 13 + .../plan/relational/sql/ast/ShowStatement.java | 17 + .../plan/relational/sql/ast/ShowSubscriptions.java | 13 + .../plan/relational/sql/ast/ShowTables.java | 12 + .../plan/relational/sql/ast/ShowTopics.java | 13 + .../plan/relational/sql/ast/ShowVariables.java | 11 + .../plan/relational/sql/ast/ShowVersion.java | 11 + .../relational/sql/ast/SimpleCaseExpression.java | 14 + .../plan/relational/sql/ast/SimpleGroupBy.java | 12 + .../plan/relational/sql/ast/SingleColumn.java | 15 + .../plan/relational/sql/ast/SkipTo.java | 12 + .../plan/relational/sql/ast/SortItem.java | 11 + .../plan/relational/sql/ast/StartPipe.java | 12 + .../plan/relational/sql/ast/Statement.java | 25 + .../plan/relational/sql/ast/StopPipe.java | 11 + .../plan/relational/sql/ast/StringLiteral.java | 11 + .../relational/sql/ast/SubqueryExpression.java | 12 + .../plan/relational/sql/ast/SubsetDefinition.java | 14 + .../plan/relational/sql/ast/SymbolReference.java | 11 + .../queryengine/plan/relational/sql/ast/Table.java | 11 + .../relational/sql/ast/TableFunctionArgument.java | 14 + .../sql/ast/TableFunctionInvocation.java | 14 + .../sql/ast/TableFunctionTableArgument.java | 15 + .../plan/relational/sql/ast/TableSubquery.java | 12 + .../plan/relational/sql/ast/TimeRange.java | 12 + .../queryengine/plan/relational/sql/ast/Trim.java | 12 + .../plan/relational/sql/ast/TypeParameter.java | 12 + .../queryengine/plan/relational/sql/ast/Union.java | 11 + .../plan/relational/sql/ast/UnloadModel.java | 14 + .../plan/relational/sql/ast/Update.java | 11 + .../plan/relational/sql/ast/UpdateAssignment.java | 13 + .../queryengine/plan/relational/sql/ast/Use.java | 11 + .../plan/relational/sql/ast/Values.java | 11 + .../relational/sql/ast/VariableDefinition.java | 13 + .../plan/relational/sql/ast/WhenClause.java | 12 + .../plan/relational/sql/ast/WindowDefinition.java | 13 + .../plan/relational/sql/ast/WindowFrame.java | 14 + .../plan/relational/sql/ast/WindowReference.java | 12 + .../relational/sql/ast/WindowSpecification.java | 18 + .../queryengine/plan/relational/sql/ast/With.java | 11 + .../plan/relational/sql/ast/WithQuery.java | 42 +- .../relational/sql/ast/WrappedInsertStatement.java | 414 ++- .../plan/relational/sql/ast/WrappedStatement.java | 5 + .../relational/sql/ast/ZeroOrMoreQuantifier.java | 12 + .../relational/sql/ast/ZeroOrOneQuantifier.java | 12 + .../plan/relational/sql/parser/AstBuilder.java | 76 +- .../plan/relational/sql/util/AstUtil.java | 30 + .../plan/relational/sql/util/SqlFormatter.java | 8 +- .../plan/relational/type/CompatibleResolver.java | 5 + .../plan/relational/type/InternalTypeManager.java | 6 + .../relational/utils/ResultColumnAppender.java | 145 ++ .../plan/relational/utils/TypeUtil.java | 13 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 2 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 6 +- .../db/queryengine/plan/statement/Statement.java | 23 + .../plan/statement/StatementVisitor.java | 12 + .../plan/statement/component/OrderByComponent.java | 6 + .../plan/statement/crud/InsertBaseStatement.java | 155 +- .../crud/InsertMultiTabletsStatement.java | 6 + .../plan/statement/crud/InsertRowStatement.java | 49 + .../crud/InsertRowsOfOneDeviceStatement.java | 6 + .../plan/statement/crud/InsertRowsStatement.java | 6 + .../plan/statement/crud/InsertTabletStatement.java | 263 +- .../plan/statement/crud/LoadTsFileStatement.java | 48 + .../InternalBatchActivateTemplateStatement.java | 2 +- .../internal/SeriesSchemaFetchStatement.java | 2 +- .../AlterTimeSeriesDataTypeStatement.java} | 35 +- .../metadata/AlterTimeSeriesStatement.java | 15 +- .../metadata/ShowAvailableUrlsStatement.java} | 28 +- .../metadata/pipe/AlterPipeStatement.java | 40 +- .../metadata/pipe/CreatePipeStatement.java | 28 +- .../template/ActivateTemplateStatement.java | 11 +- .../template/BatchActivateTemplateStatement.java | 2 +- .../template/DeactivateTemplateStatement.java | 2 +- .../db/queryengine/plan/udf/UDTFForecast.java | 27 +- .../FragmentInstanceStatisticsDrawer.java | 28 +- .../binary/ReadObject2ColumnTransformer.java | 78 + .../ternary/ReadObject3ColumnTransformer.java | 99 + .../udf/UserDefineScalarFunctionTransformer.java | 2 +- .../AbstractCastFunctionColumnTransformer.java | 10 + ...r.java => AbstractLengthColumnTransformer.java} | 15 +- .../unary/scalar/BlobLengthColumnTransformer.java} | 26 +- .../scalar/CastFunctionColumnTransformer.java | 3 + .../unary/scalar/LengthColumnTransformer.java | 30 +- .../scalar/ObjectLengthColumnTransformer.java} | 23 +- .../unary/scalar/ReadObjectColumnTransformer.java | 126 + .../scalar/TryCastFunctionColumnTransformer.java | 3 + .../dag/input/ConstantInputReader.java | 1 + .../unary/ArithmeticNegationTransformer.java | 1 + .../dag/transformer/unary/InTransformer.java | 2 + .../unary/scalar/CastFunctionTransformer.java | 7 + .../unary/scalar/DiffFunctionTransformer.java | 1 + .../unary/scalar/RoundFunctionTransformer.java | 1 + .../transformation/dag/util/TransformUtils.java | 2 + .../transformation/dag/util/TypeUtils.java | 4 +- .../datastructure/row/SerializableRowList.java | 2 + .../datastructure/tv/SerializableTVList.java | 1 + .../rescon/MemSchemaRegionStatistics.java | 2 +- .../schemaengine/schemaregion/ISchemaRegion.java | 12 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 26 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 19 +- .../schemaregion/mtree/IMTreeStore.java | 2 +- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 47 +- .../schemaregion/mtree/impl/mem/MemMTreeStore.java | 2 +- .../mtree/impl/mem/mnode/info/LogicalViewInfo.java | 3 + .../mtree/impl/mem/mnode/info/MeasurementInfo.java | 3 + .../mnode/iterator/AbstractTraverserIterator.java | 2 +- .../mnode/iterator/MemoryTraverserIterator.java | 2 +- .../mtree/impl/pbtree/CachedMTreeStore.java | 2 +- .../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 2 +- .../pbtree/ReentrantReadOnlyCachedMTreeStore.java | 2 +- .../mnode/iterator/CachedTraverserIterator.java | 2 +- .../schemaregion/mtree/traverser/Traverser.java | 2 +- .../schemaregion/read/req/IShowTimeSeriesPlan.java | 2 +- .../read/req/SchemaRegionReadPlanFactory.java | 2 +- .../read/req/impl/ShowTimeSeriesPlanImpl.java | 2 +- .../read/resp/info/impl/ShowDevicesResult.java | 24 +- .../schemaregion/utils/MNodeUtils.java | 2 +- .../schemaregion/utils/ResourceByPathUtils.java | 39 +- .../db/schemaengine/table/DataNodeTableCache.java | 17 +- .../template/ClusterTemplateManager.java | 1 + .../db/schemaengine/template/ITemplateManager.java | 1 + .../template/TemplateInternalRPCUtil.java | 2 + .../java/org/apache/iotdb/db/service/DataNode.java | 3 + .../db/service/metrics/CompactionMetrics.java | 14 +- .../iotdb/db/service/metrics/FileMetrics.java | 20 + .../iotdb/db/service/metrics/WritingMetrics.java | 3 +- .../db/service/metrics/file/ObjectFileMetrics.java | 86 + .../metrics/memory/QueryEngineMemoryMetrics.java | 12 +- .../iotdb/db/storageengine/StorageEngine.java | 60 +- .../storageengine/dataregion/Base32ObjectPath.java | 169 ++ .../db/storageengine/dataregion/DataRegion.java | 738 +++++- .../dataregion/IDataRegionForQuery.java | 2 +- .../db/storageengine/dataregion/IObjectPath.java | 70 + .../storageengine/dataregion/PlainObjectPath.java | 126 + .../dataregion/VirtualDataRegion.java | 2 +- .../performer/impl/FastCompactionPerformer.java | 32 +- .../impl/ReadChunkCompactionPerformer.java | 7 +- .../execute/task/AbstractCompactionTask.java | 13 + .../execute/task/InnerSpaceCompactionTask.java | 3 + .../execute/task/SettleCompactionTask.java | 11 +- .../subtask/FastCompactionPerformerSubTask.java | 33 +- .../execute/utils/CompactionSeriesContext.java | 60 + .../compaction/execute/utils/CompactionUtils.java | 143 + .../execute/utils/MultiTsFileDeviceIterator.java | 75 +- .../fast/FastAlignedSeriesCompactionExecutor.java | 33 +- .../FastNonAlignedSeriesCompactionExecutor.java | 53 +- .../fast/element/ChunkMetadataElement.java | 10 +- .../fast/reader/CompactionAlignedChunkReader.java | 8 +- .../ReadChunkAlignedSeriesCompactionExecutor.java | 31 +- .../readchunk/SingleSeriesCompactionExecutor.java | 21 +- .../utils/writer/AbstractCompactionWriter.java | 1 + .../compaction/repair/RepairTimePartition.java | 2 +- .../schedule/CompactionScheduleContext.java | 27 + .../compaction/schedule/TTLScheduleTask.java | 12 + .../selector/utils/TsFileResourceCandidate.java | 7 +- .../dataregion/memtable/AbstractMemTable.java | 98 +- .../memtable/AbstractWritableMemChunk.java | 4 +- .../memtable/AlignedReadOnlyMemChunk.java | 39 +- .../memtable/AlignedWritableMemChunk.java | 199 +- .../memtable/AlignedWritableMemChunkGroup.java | 7 + .../dataregion/memtable/IMemTable.java | 10 +- .../memtable/IWritableMemChunkGroup.java | 4 + .../dataregion/memtable/ReadOnlyMemChunk.java | 37 +- .../dataregion/memtable/TsFileProcessor.java | 40 +- .../dataregion/memtable/WritableMemChunk.java | 87 +- .../dataregion/memtable/WritableMemChunkGroup.java | 18 + .../dataregion/modification/DeletionPredicate.java | 8 + .../modification/TableDeletionEntry.java | 9 + .../read/reader/chunk/DiskAlignedChunkLoader.java | 16 +- .../read/reader/chunk/DiskChunkLoader.java | 17 + .../read/reader/chunk/MemAlignedPageReader.java | 10 +- .../read/reader/chunk/MemPageReader.java | 2 + .../dataregion/snapshot/SnapshotTaker.java | 14 +- .../dataregion/tsfile/TsFileResource.java | 13 +- .../tsfile/timeindex/ArrayDeviceTimeIndex.java | 5 +- .../dataregion/tsfile/timeindex/FileTimeIndex.java | 3 +- .../dataregion/tsfile/timeindex/ITimeIndex.java | 10 +- .../tsfile/timeindex/PlainDeviceTimeIndex.java | 3 +- .../allocation/AbstractNodeAllocationStrategy.java | 21 +- .../dataregion/wal/buffer/WALBuffer.java | 3 + .../dataregion/wal/buffer/WALEntry.java | 6 + .../dataregion/wal/buffer/WALEntryType.java | 4 +- .../dataregion/wal/buffer/WALInfoEntry.java | 4 + .../dataregion/wal/node/IWALNode.java | 3 + .../dataregion/wal/node/WALFakeNode.java | 6 + .../storageengine/dataregion/wal/node/WALNode.java | 52 +- .../file/UnsealedTsFileRecoverPerformer.java | 8 +- .../db/storageengine/load/LoadTsFileManager.java | 26 +- .../storageengine/load/active/ActiveLoadAgent.java | 90 + ...leStatementDataTypeConvertExecutionVisitor.java | 2 +- .../load/memory/LoadTsFileMemoryManager.java | 19 +- .../splitter/BatchedAlignedValueChunkData.java | 2 + .../db/storageengine/rescon/disk/TierManager.java | 99 + .../rescon/memory/PrimitiveArrayManager.java | 1 + .../iotdb/db/tools/TsFileResourcePrinter.java | 5 +- .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 1 + .../org/apache/iotdb/db/utils/CommonUtils.java | 23 + .../iotdb/db/utils/EncodingInferenceUtils.java | 1 + .../java/org/apache/iotdb/db/utils/MemUtils.java | 4 +- .../org/apache/iotdb/db/utils/ObjectTypeUtils.java | 335 +++ .../org/apache/iotdb/db/utils/ObjectWriter.java | 82 + .../apache/iotdb/db/utils/QueryDataSetUtils.java | 11 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 338 +++ .../org/apache/iotdb/db/utils/TabletDecoder.java | 1 + .../apache/iotdb/db/utils/TimeValuePairUtils.java | 2 + .../apache/iotdb/db/utils/TypeInferenceUtils.java | 1 + .../apache/iotdb/db/utils/cte/CteDataReader.java | 52 + .../apache/iotdb/db/utils/cte/CteDataStore.java | 109 + .../apache/iotdb/db/utils/cte/MemoryReader.java | 75 + .../db/utils/datastructure/AlignedTVList.java | 182 +- .../db/utils/datastructure/BackIntTVList.java | 7 + .../iotdb/db/utils/datastructure/IntTVList.java | 35 +- .../MergeSortMultiAlignedTVListIterator.java | 1 + .../MergeSortMultiTVListIterator.java | 1 + .../datastructure/MultiAlignedTVListIterator.java | 1 + .../utils/datastructure/MultiTVListIterator.java | 1 + .../db/utils/datastructure/QuickIntTVList.java | 7 + .../iotdb/db/utils/datastructure/TVList.java | 41 +- .../iotdb/db/utils/datastructure/TimIntTVList.java | 7 + .../db/utils/windowing/window/WindowImpl.java | 1 + .../apache/iotdb/db/auth/AuthorityCheckerTest.java | 51 + .../org/apache/iotdb/db/auth/TreeAccessTest.java | 84 + .../schemaregion/SchemaExecutionVisitorTest.java | 277 ++ .../cache/TreeDeviceSchemaCacheManagerTest.java | 16 +- .../schemaRegion/SchemaRegionManagementTest.java | 2 +- .../SchemaRegionSimpleRecoverTest.java | 2 +- .../schemaRegion/SchemaRegionTemplateTest.java | 2 +- .../schemaRegion/SchemaRegionTestUtil.java | 2 +- .../schemaRegion/SchemaStatisticsTest.java | 2 +- .../pipe/event/PipeTabletInsertionEventTest.java | 91 +- .../pipe/event/PipeTsFileInsertionEventTest.java | 298 +++ .../pipe/event/TsFileInsertionEventParserTest.java | 41 +- .../db/pipe/pattern/IoTDBTreePatternTest.java | 36 + .../db/pipe/pattern/TreePatternPruningTest.java | 156 ++ .../pipe/sink/PipeDataNodeThriftRequestTest.java | 4 +- .../apache/iotdb/db/pipe/sink/PipeSinkTest.java | 102 + .../db/pipe/sink/PipeStatementEventSorterTest.java | 313 +++ .../sink/util/TabletStatementConverterTest.java | 607 +++++ .../source/PipeTreeStatementToPlanVisitorTest.java | 87 + .../common/schematree/ClusterSchemaTreeTest.java | 2 +- .../fragment/FragmentInstanceExecutionTest.java | 243 +- .../execution/operator/CteScanOperatorTest.java | 246 ++ .../operator/DeviceViewIntoOperatorTest.java | 487 ++++ .../operator/MergeTreeSortOperatorTest.java | 5 + .../execution/operator/TreeIntoOperatorTest.java | 303 +++ .../process/window/TableWindowOperatorTest.java | 51 + .../predicate/TreePredicateConversionTest.java | 572 ++++ .../plan/analyze/FakeSchemaFetcherImpl.java | 2 +- .../plan/function/RecordObjectTypeTest.java | 156 ++ .../plan/parser/StatementGeneratorTest.java | 33 + .../plan/planner/distribution/LastQueryTest.java | 57 + .../plan/planner/distribution/Util.java | 2 +- .../plan/planner/distribution/Util2.java | 2 +- ...NodeSerdeTest.java => SourceNodeSerdeTest.java} | 20 +- .../statement/sys/pipe/PipeStatementTest.java | 8 +- .../plan/relational/analyzer/AnalyzerTest.java | 47 +- .../plan/relational/analyzer/AuthTest.java | 10 +- .../analyzer/EvaluateEmptyIntersectTest.java | 48 + .../plan/relational/analyzer/ExceptTest.java | 132 + .../relational/analyzer/InsertIntoQueryTest.java | 7 + .../analyzer/LimitOffsetPushDownTest.java | 7 + .../plan/relational/analyzer/MergeExceptTest.java | 147 ++ .../relational/analyzer/MergeIntersectTest.java | 112 + .../analyzer/PushAggregationThroughUnionTest.java | 6 + .../analyzer/RemoveEmptyExceptBranchesTest.java | 101 + .../analyzer/RowPatternRecognitionTest.java | 4 +- .../plan/relational/analyzer/SortTest.java | 7 + .../plan/relational/analyzer/SubQueryTest.java | 7 + .../plan/relational/analyzer/TSBSMetadata.java | 19 +- .../relational/analyzer/TableFunctionTest.java | 7 +- .../plan/relational/analyzer/TestMetadata.java | 44 +- .../fetcher/cache/TableDeviceSchemaCacheTest.java | 9 + .../relational/planner/CteMaterializerTest.java | 344 +++ .../plan/relational/planner/CteSubqueryTest.java | 168 ++ .../plan/relational/planner/PlanTester.java | 43 +- .../planner/UncorrelatedSubqueryTest.java | 261 +- .../planner/assertions/AliasMatcher.java | 8 + .../planner/assertions/ColumnReference.java | 23 +- .../planner/assertions/CteScanMatcher.java | 86 + .../planner/assertions/DeviceTableScanMatcher.java | 11 + .../planner/assertions/PlanMatchPattern.java | 48 +- .../planner/assertions/RvalueMatcher.java | 8 + .../planner/assertions/TableScanMatcher.java | 17 + .../informationschema/CurrentQueriesTest.java | 184 ++ .../informationschema}/ShowQueriesTest.java | 8 +- .../predicate/TablePredicateConversionTest.java | 506 ++++ .../relational/sql/AstMemoryEstimatorTest.java | 892 +++++++ .../plan/statement/InsertStatementTest.java | 97 +- .../scalar/BlobLengthColumnTransformerTest.java | 148 ++ .../unary/scalar/ObjectTypeFunctionTest.java | 186 ++ .../DataNodeInternalRPCServiceImplTest.java | 59 +- .../iotdb/db/storageengine/StorageEngineTest.java | 4 +- .../storageengine/dataregion/DataRegionTest.java | 4 +- .../compaction/CompactionDataTypeAlterTest.java | 322 +++ ...ctionDataTypeNotMatchAlterableDataTypeTest.java | 221 ++ .../compaction/CompactionSchedulerTest.java | 35 + .../FastNonAlignedCrossCompactionTest.java | 5 + .../ReadPointNonAlignedCrossCompactionTest.java | 4 + .../inner/InnerSpaceCompactionExceptionTest.java | 16 + .../object/ObjectTypeCompactionTest.java | 459 ++++ .../repair/RepairUnsortedFileSchedulerTest.java | 10 +- .../memtable/AlignedTVListIteratorTest.java | 57 + .../memtable/WritableMemChunkRegionScanTest.java | 445 ++++ .../dataregion/snapshot/IoTDBSnapshotTest.java | 2 +- .../wal/allocation/FirstCreateStrategyTest.java | 81 + .../db/utils/SchemaRegionSnapshotParserTest.java | 2 +- .../org/apache/iotdb/db/utils/SchemaUtilsTest.java | 174 ++ .../apache/iotdb/db/utils/TSDataTypeTestUtils.java | 111 + .../db/utils/datastructure/AlignedTVListTest.java | 10 +- .../db/utils/datastructure/IntTVListTest.java | 11 +- .../datastructure/PrimitiveArrayManagerTest.java | 13 +- iotdb-core/metrics/core/pom.xml | 4 +- iotdb-core/metrics/interface/pom.xml | 8 +- iotdb-core/metrics/pom.xml | 2 +- iotdb-core/node-commons/pom.xml | 29 +- .../conf/iotdb-system.properties.template | 31 + .../iotdb/commons/audit/AbstractAuditLogger.java | 89 +- .../iotdb/commons/auth/entity/PrivilegeUnion.java | 2 +- .../apache/iotdb/commons/client/ClientManager.java | 25 + .../iotdb/commons/client/ClientPoolFactory.java | 29 + .../async/AsyncAINodeInternalServiceClient.java} | 25 +- .../async/AsyncPipeDataTransferServiceClient.java | 21 +- .../iotdb/commons/concurrent/ThreadName.java | 1 + .../apache/iotdb/commons/conf/CommonConfig.java | 60 + .../iotdb/commons/conf/CommonDescriptor.java | 11 + .../apache/iotdb/commons/conf/IoTDBConstant.java | 6 + .../commons/exception/ObjectFileNotExist.java} | 15 +- .../commons/file/SystemPropertiesHandler.java | 13 +- .../iotdb/commons/model/ModelInformation.java | 43 +- .../org/apache/iotdb/commons/model/ModelTable.java | 4 +- .../apache/iotdb/commons/path/MeasurementPath.java | 31 + .../service/PipePluginExecutableManager.java | 9 + .../task/subtask/PipeAbstractSinkSubtask.java | 13 + .../iotdb/commons/pipe/config/PipeConfig.java | 10 + .../iotdb/commons/pipe/config/PipeDescriptor.java | 16 + .../pipe/config/constant/PipeSinkConstant.java | 49 +- .../datastructure/pattern/IoTDBTreePattern.java | 31 + .../pipe/datastructure/pattern/TreePattern.java | 490 +++- .../pattern/UnionIoTDBTreePattern.java | 6 + .../pattern/WithExclusionIoTDBTreePattern.java | 2 - .../pattern/WithExclusionTreePattern.java | 2 - .../commons/pipe/receiver/IoTDBFileReceiver.java | 11 +- .../pipe/receiver/PipeReceiverStatusHandler.java | 39 +- .../pipe/source/IoTDBNonDataRegionSource.java | 14 +- .../iotdb/commons/pipe/source/IoTDBSource.java | 2 + .../iotdb/commons/schema/column/ColumnHeader.java | 14 +- .../schema/column/ColumnHeaderConstant.java | 28 +- .../iotdb/commons/schema/filter/SchemaFilter.java | 3 +- .../commons/schema/filter/impl/DataTypeFilter.java | 9 + .../schema/filter/impl/PathContainsFilter.java | 10 + .../commons/schema/filter/impl/TagFilter.java | 9 + .../commons/schema/filter/impl/TemplateFilter.java | 10 + .../commons/schema/filter/impl/ViewTypeFilter.java | 10 + .../multichildren/AbstractMultiChildrenFilter.java | 25 + .../filter/impl/multichildren/AndFilter.java | 9 + .../schema/filter/impl/multichildren/OrFilter.java | 9 + .../singlechild/AbstractSingleChildFilter.java | 16 + .../filter/impl/singlechild/AttributeFilter.java | 10 + .../schema/filter/impl/singlechild/NotFilter.java | 9 + .../schema/filter/impl/singlechild/TagFilter.java | 10 + .../filter/impl/values/ComparisonFilter.java | 10 + .../schema/filter/impl/values/InFilter.java | 8 + .../schema/filter/impl/values/LikeFilter.java | 11 + .../schema/filter/impl/values/PreciseFilter.java | 10 + .../table/AlterOrDropTableOperationType.java | 5 +- .../commons/schema/table/InformationSchema.java | 78 +- .../schema/table/InsertNodeMeasurementInfo.java | 244 ++ .../schema/table/TsFileTableSchemaUtil.java | 207 ++ .../apache/iotdb/commons/schema/table/TsTable.java | 288 ++- .../schema/table/column/AttributeColumnSchema.java | 7 + .../schema/table/column/FieldColumnSchema.java | 15 +- .../schema/table/column/TagColumnSchema.java | 6 + .../schema/table/column/TimeColumnSchema.java | 6 + .../schema/table/column/TsTableColumnSchema.java | 12 + .../table/column/TsTableColumnSchemaUtil.java | 12 + .../iotdb/commons/schema}/template/Template.java | 2 +- .../schema/tree/AlterTimeSeriesOperationType.java} | 29 +- .../apache/iotdb/commons/udf/builtin/UDTFAbs.java | 3 + .../iotdb/commons/udf/builtin/UDTFBottomK.java | 1 + .../commons/udf/builtin/UDTFCommonDerivative.java | 1 + .../udf/builtin/UDTFCommonValueDifference.java | 1 + .../iotdb/commons/udf/builtin/UDTFConst.java | 5 + .../udf/builtin/UDTFContinuouslySatisfy.java | 2 + .../udf/builtin/UDTFEqualSizeBucketAggSample.java | 1 + .../udf/builtin/UDTFEqualSizeBucketM4Sample.java | 1 + .../builtin/UDTFEqualSizeBucketOutlierSample.java | 1 + .../builtin/UDTFEqualSizeBucketRandomSample.java | 1 + .../iotdb/commons/udf/builtin/UDTFInRange.java | 3 + .../apache/iotdb/commons/udf/builtin/UDTFJexl.java | 3 + .../apache/iotdb/commons/udf/builtin/UDTFM4.java | 1 + .../apache/iotdb/commons/udf/builtin/UDTFMath.java | 3 + .../udf/builtin/UDTFNonNegativeDerivative.java | 1 + .../builtin/UDTFNonNegativeValueDifference.java | 1 + .../iotdb/commons/udf/builtin/UDTFOnOff.java | 1 + .../iotdb/commons/udf/builtin/UDTFSelectK.java | 2 + .../apache/iotdb/commons/udf/builtin/UDTFTopK.java | 1 + .../iotdb/commons/udf/builtin/UDTFValueTrend.java | 1 + .../relational/TableBuiltinScalarFunction.java | 1 + .../commons/udf/utils/UDFDataTypeTransformer.java | 7 + .../apache/iotdb/commons/utils/JVMCommonUtils.java | 7 +- .../apache/iotdb/commons/utils/MetadataUtils.java} | 15 +- .../apache/iotdb/commons/utils/SerializeUtils.java | 6 + .../apache/iotdb/commons/utils/WindowsOSUtils.java | 64 + .../PipeReceiverStatusHandlerTest.java | 302 +++ .../commons/pipe/task/PipeSleepIntervalTest.java | 83 + .../schema/table/TsFileTableSchemaUtilTest.java | 748 ++++++ .../iotdb/commons/utils/WindowsOSUtilsTest.java} | 40 +- iotdb-core/pom.xml | 2 +- iotdb-core/relational-grammar/pom.xml | 2 +- .../db/relational/grammar/sql/RelationalSql.g4 | 37 +- iotdb-protocol/openapi/pom.xml | 2 +- iotdb-protocol/pom.xml | 2 +- iotdb-protocol/thrift-ainode/pom.xml | 4 +- .../thrift-ainode/src/main/thrift/ainode.thrift | 35 +- iotdb-protocol/thrift-commons/pom.xml | 2 +- iotdb-protocol/thrift-confignode/pom.xml | 4 +- .../src/main/thrift/confignode.thrift | 78 +- iotdb-protocol/thrift-consensus/pom.xml | 4 +- iotdb-protocol/thrift-datanode/pom.xml | 4 +- .../thrift-datanode/src/main/thrift/client.thrift | 3 +- .../src/main/thrift/datanode.thrift | 24 +- library-udf/pom.xml | 4 +- pom.xml | 44 +- scripts/sbin/windows/start-confignode.bat | 28 - scripts/sbin/windows/start-datanode.bat | 48 - 1238 files changed, 59573 insertions(+), 16881 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index ffaeb60bab0,d34abd60946..2fe30a0fcae --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@@ -19,9 -19,9 +19,10 @@@ package org.apache.iotdb.db.queryengine.execution.operator.source; +import java.util.stream.Collectors; import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; + import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; @@@ -35,10 -35,12 +36,13 @@@ import org.apache.iotdb.db.storageengin import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.MemAlignedChunkMetadataLoader; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.MemChunkMetadataLoader; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.utils.ModificationUtils; + import org.apache.iotdb.db.utils.SchemaUtils; + import org.apache.tsfile.enums.TSDataType; + import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@@ -56,7 -59,7 +61,8 @@@ import java.util.ArrayList import java.util.Collections; import java.util.List; import java.util.Set; +import org.apache.tsfile.write.schema.IMeasurementSchema; + import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@@ -117,9 -113,11 +123,10 @@@ public class FileLoaderUtils context.isDebug(), context); if (timeSeriesMetadata != null) { + SchemaUtils.changeMetadataModified(timeSeriesMetadata, seriesPath.getSeriesType()); long t2 = System.nanoTime(); List<ModEntry> pathModifications = - context.getPathModifications( - resource, seriesPath.getDeviceId(), seriesPath.getMeasurement()); + context.getPathModifications(resource, deviceId, measurement); timeSeriesMetadata.setModified(!pathModifications.isEmpty()); timeSeriesMetadata.setChunkMetadataLoader( new DiskChunkMetadataLoader(resource, context, globalTimeFilter, pathModifications)); @@@ -204,7 -207,8 +217,8 @@@ if (resource.isClosed()) { alignedTimeSeriesMetadata = loadAlignedTimeSeriesMetadataFromDisk( - resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows); + resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows, maxTsFileSetEndVersion); + SchemaUtils.changeAlignedMetadataModified(alignedTimeSeriesMetadata, targetDataTypeList); } else { // if the tsfile is unclosed, we just get it directly from TsFileResource loadFromMem = true; alignedTimeSeriesMetadata = diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 95a121e61b4,6ebfa6dcfdc..bc8a992cbdd --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@@ -321,8 -322,7 +324,9 @@@ public enum PlanNodeType RELATIONAL_INSERT_ROW((short) 2001), RELATIONAL_INSERT_ROWS((short) 2002), RELATIONAL_DELETE_DATA((short) 2003), - EVOLVE_SCHEMA((short) 2004), - PIPE_ENRICHED_EVOLVE_SCHEMA((short) 2005), + OBJECT_FILE_NODE((short) 2004), ++ EVOLVE_SCHEMA((short) 2005), ++ PIPE_ENRICHED_EVOLVE_SCHEMA((short) 2006), ; public static final int BYTES = Short.BYTES; @@@ -367,9 -367,7 +371,11 @@@ case 2003: return RelationalDeleteDataNode.deserializeFromWAL(stream); case 2004: + return ObjectNode.deserializeFromWAL(stream); ++ case 2005 + return EvolveSchemaNode.deserializeFromWAL(stream); - case 2005: ++ case 2006: + return PipeEnrichedEvolveSchemaNode.deserializeFromWAL(stream); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } @@@ -397,9 -395,7 +403,11 @@@ case 2003: return RelationalDeleteDataNode.deserializeFromWAL(buffer); case 2004: - return EvolveSchemaNode.deserializeFromWAL(buffer); + return ObjectNode.deserialize(buffer); + case 2005: ++ return EvolveSchemaNode.deserializeFromWAL(buffer); ++ case 2006: + return PipeEnrichedEvolveSchemaNode.deserializeFromWAL(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } @@@ -726,9 -724,7 +736,11 @@@ case 2003: return RelationalDeleteDataNode.deserialize(buffer); case 2004: - return EvolveSchemaNode.deserialize(buffer); + return ObjectNode.deserialize(buffer); + case 2005: ++ return EvolveSchemaNode.deserialize(buffer); ++ case 2006: + return PipeEnrichedEvolveSchemaNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index d9218b34774,17e24fc23c2..e64b6ba0240 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@@ -634,10 -643,10 +645,14 @@@ public abstract class PlanVisitor<R, C return visitPlan(node, context); } + public R visitEvolveSchemaNode(EvolveSchemaNode node, C context) { + return visitPlan(node, context); + } + + public R visitWriteObjectFile(ObjectNode node, C context) { + return visitPlan(node, context); + } + ///////////////////////////////////////////////////////////////////////////////////////////////// // Pipe Related Node ///////////////////////////////////////////////////////////////////////////////////////////////// diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 201007dcdb3,746e042ab14..d57e7b2a904 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@@ -111,8 -116,8 +116,9 @@@ import org.apache.iotdb.db.storageengin import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessorInfo; + import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry.ModType; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; @@@ -170,11 -169,14 +178,15 @@@ import org.apache.iotdb.rpc.TSStatusCod import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; + import com.google.common.io.BaseEncoding; import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.io.FileUtils; + import org.apache.tsfile.external.commons.lang3.tuple.Triple; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; + import org.apache.tsfile.file.metadata.IDeviceID.Factory; + import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.fileSystem.FSType; import org.apache.tsfile.fileSystem.fsFactory.FSFactory; @@@ -3097,99 -3230,108 +3430,181 @@@ public class DataRegion implements IDat } } + private boolean canBeFullyDeleted( + ArrayDeviceTimeIndex deviceTimeIndex, TableDeletionEntry tableDeletionEntry) { + Set<IDeviceID> devicesInFile = deviceTimeIndex.getDevices(); + String tableName = tableDeletionEntry.getTableName(); + long matchSize = + devicesInFile.stream() + .filter( + device -> { + if (logger.isDebugEnabled()) { + logger.debug( + "device is {}, deviceTable is {}, tableDeletionEntry.getPredicate().matches(device) is {}", + device, + device.getTableName(), + tableDeletionEntry.getPredicate().matches(device)); + } + return tableName.equals(device.getTableName()) + && tableDeletionEntry.getPredicate().matches(device); + }) + .count(); + boolean onlyOneTable = matchSize == devicesInFile.size(); + if (logger.isDebugEnabled()) { + logger.debug( + "tableName is {}, matchSize is {}, onlyOneTable is {}", + tableName, + matchSize, + onlyOneTable); + } + + if (onlyOneTable) { + matchSize = 0; + for (IDeviceID device : devicesInFile) { + Optional<Long> optStart = deviceTimeIndex.getStartTime(device); + Optional<Long> optEnd = deviceTimeIndex.getEndTime(device); + if (!optStart.isPresent() || !optEnd.isPresent()) { + continue; + } + + long fileStartTime = optStart.get(); + long fileEndTime = optEnd.get(); + + if (logger.isDebugEnabled()) { + logger.debug( + "tableName is {}, device is {}, deletionStartTime is {}, deletionEndTime is {}, fileStartTime is {}, fileEndTime is {}", + device.getTableName(), + device, + tableDeletionEntry.getStartTime(), + tableDeletionEntry.getEndTime(), + fileStartTime, + fileEndTime); + } + if (isFileFullyMatchedByTime(tableDeletionEntry, fileStartTime, fileEndTime)) { + ++matchSize; + } else { + return false; + } + } + return matchSize == devicesInFile.size(); + } else { + return false; + } + } + private void deleteDataInSealedFiles(Collection<TsFileResource> sealedTsFiles, ModEntry deletion) throws IOException { - Set<ModificationFile> involvedModificationFiles = new HashSet<>(); - List<TsFileResource> deletedByMods = new ArrayList<>(); + Set<Pair<ModificationFile, ModEntry>> involvedModificationFiles = new HashSet<>(); List<TsFileResource> deletedByFiles = new ArrayList<>(); + boolean isDropMeasurementExist = false; + IDPredicate.IDPredicateType idPredicateType = null; + + if (deletion instanceof TableDeletionEntry) { + TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; + isDropMeasurementExist = !tableDeletionEntry.getPredicate().getMeasurementNames().isEmpty(); + idPredicateType = tableDeletionEntry.getPredicate().getIdPredicateType(); + } + for (TsFileResource sealedTsFile : sealedTsFiles) { if (canSkipDelete(sealedTsFile, deletion)) { continue; } ITimeIndex timeIndex = sealedTsFile.getTimeIndex(); + EvolvedSchema evolvedSchema = sealedTsFile.getMergedEvolvedSchema(); if ((timeIndex instanceof ArrayDeviceTimeIndex) - && (deletion.getType() == ModEntry.ModType.TABLE_DELETION)) { + && (deletion.getType() == ModType.TABLE_DELETION)) { ArrayDeviceTimeIndex deviceTimeIndex = (ArrayDeviceTimeIndex) timeIndex; ++ + TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; + tableDeletionEntry = + evolvedSchema != null + ? evolvedSchema.rewriteToOriginal(tableDeletionEntry) + : tableDeletionEntry; - if (canBeFullyDeleted(deviceTimeIndex, tableDeletionEntry)) { - deletedByFiles.add(sealedTsFile); ++ + Set<IDeviceID> devicesInFile = deviceTimeIndex.getDevices(); + boolean onlyOneTable = false; + + if (deletion instanceof TableDeletionEntry) { + TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; + String tableName = tableDeletionEntry.getTableName(); + long matchSize = + devicesInFile.stream() + .filter( + device -> { + if (logger.isDebugEnabled()) { + logger.debug( + "device is {}, deviceTable is {}, tableDeletionEntry.getPredicate().matches(device) is {}", + device, + device.getTableName(), + tableDeletionEntry.getPredicate().matches(device)); + } + return tableName.equals(device.getTableName()) + && tableDeletionEntry.getPredicate().matches(device); + }) + .count(); + onlyOneTable = matchSize == devicesInFile.size(); + if (logger.isDebugEnabled()) { + logger.debug( + "tableName is {}, matchSize is {}, onlyOneTable is {}", + tableName, + matchSize, + onlyOneTable); + } + } + + if (onlyOneTable) { + int matchSize = 0; + for (IDeviceID device : devicesInFile) { + Optional<Long> optStart = deviceTimeIndex.getStartTime(device); + Optional<Long> optEnd = deviceTimeIndex.getEndTime(device); + if (!optStart.isPresent() || !optEnd.isPresent()) { + continue; + } + + long fileStartTime = optStart.get(); + long fileEndTime = optEnd.get(); + + if (logger.isDebugEnabled()) { + logger.debug( + "tableName is {}, device is {}, deletionStartTime is {}, deletionEndTime is {}, fileStartTime is {}, fileEndTime is {}", + device.getTableName(), + device, + deletion.getStartTime(), + deletion.getEndTime(), + fileStartTime, + fileEndTime); + } + if (isFileFullyMatchedByTime(deletion, fileStartTime, fileEndTime) + && idPredicateType.equals(IDPredicate.IDPredicateType.NOP) + && !isDropMeasurementExist) { + ++matchSize; + } else { + deletedByMods.add(sealedTsFile); + break; + } + } + if (matchSize == devicesInFile.size()) { + deletedByFiles.add(sealedTsFile); + } + + if (logger.isDebugEnabled()) { + logger.debug("expect is {}, actual is {}", devicesInFile.size(), matchSize); + for (TsFileResource tsFileResource : deletedByFiles) { + logger.debug( + "delete tsFileResource is {}", tsFileResource.getTsFile().getAbsolutePath()); + } + } } else { - involvedModificationFiles.add(sealedTsFile.getModFileForWrite()); + involvedModificationFiles.add( + new Pair<>(sealedTsFile.getModFileForWrite(), tableDeletionEntry)); } } else { - involvedModificationFiles.add(sealedTsFile.getModFileForWrite()); + involvedModificationFiles.add( + new Pair<>( + sealedTsFile.getModFileForWrite(), + evolvedSchema != null ? evolvedSchema.rewriteToOriginal(deletion) : deletion)); } } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index 2acf0afa17f,54b21ddd382..b531a6460d0 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java @@@ -207,21 -184,10 +210,21 @@@ public class FastCompactionPerforme sortedSourceFiles.addAll(unseqFiles); boolean isTreeModel = !isAligned || device.getTableName().startsWith("root."); long ttl = deviceIterator.getTTLForCurrentDevice(); - sortedSourceFiles.removeIf(x -> x.definitelyNotContains(device)); + sortedSourceFiles.removeIf( + x -> { + EvolvedSchema evolvedSchema = x.getMergedEvolvedSchema( + maxTsFileSetEndVersionAndMinResource.left); + IDeviceID originalDevice = device; + if (evolvedSchema != null) { + originalDevice = evolvedSchema.rewriteToOriginal(device); + } - return x.definitelyNotContains(originalDevice) || !x.isDeviceAlive(originalDevice, ttl); ++ return x.definitelyNotContains(originalDevice); + }); // checked above - //noinspection OptionalGetWithoutIsPresent - sortedSourceFiles.sort(Comparator.comparingLong(x -> x.getStartTime(device).get())); + sortedSourceFiles.sort(Comparator.comparingLong(x -> { + //noinspection OptionalGetWithoutIsPresent + return x.getStartTime(device, maxTsFileSetEndVersionAndMinResource.left).get(); + })); ModEntry ttlDeletion = null; if (ttl != Long.MAX_VALUE) { ttlDeletion = diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/FastCompactionPerformerSubTask.java index 1050e40bd56,873993d97df..3de4656b66c --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/FastCompactionPerformerSubTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/FastCompactionPerformerSubTask.java @@@ -76,8 -76,8 +77,10 @@@ public class FastCompactionPerformerSub private List<IMeasurementSchema> measurementSchemas; + private final Pair<Long, TsFileResource> maxTsFileSetEndVersionAndMinResource; + + private Map<String, CompactionSeriesContext> compactionSeriesContextMap; + /** Used for nonAligned timeseries. */ @SuppressWarnings("squid:S107") public FastCompactionPerformerSubTask( @@@ -103,9 -102,33 +106,34 @@@ this.measurements = measurements; this.summary = summary; this.ignoreAllNullRows = true; + this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; } + public FastCompactionPerformerSubTask( + Map<String, CompactionSeriesContext> compactionSeriesContextMap, + AbstractCompactionWriter compactionWriter, + Map<TsFileResource, TsFileSequenceReader> readerCacheMap, + Map<String, PatternTreeMap<ModEntry, PatternTreeMapFactory.ModsSerializer>> + modificationCacheMap, + List<TsFileResource> sortedSourceFiles, + List<String> measurements, + IDeviceID deviceId, + FastCompactionTaskSummary summary, + int subTaskId) { + this.compactionWriter = compactionWriter; + this.subTaskId = subTaskId; + this.compactionSeriesContextMap = compactionSeriesContextMap; + this.timeseriesMetadataOffsetMap = null; + this.isAligned = false; + this.deviceId = deviceId; + this.readerCacheMap = readerCacheMap; + this.modificationCacheMap = modificationCacheMap; + this.sortedSourceFiles = sortedSourceFiles; + this.measurements = measurements; + this.summary = summary; + this.ignoreAllNullRows = true; + } + /** Used for aligned timeseries. */ public FastCompactionPerformerSubTask( AbstractCompactionWriter compactionWriter, @@@ -144,10 -166,12 +172,13 @@@ sortedSourceFiles, deviceId, subTaskId, - summary); + summary, + maxTsFileSetEndVersionAndMinResource); for (String measurement : measurements) { - seriesCompactionExecutor.setNewMeasurement(timeseriesMetadataOffsetMap.get(measurement)); + seriesCompactionExecutor.setNewMeasurement( + compactionSeriesContextMap.get(measurement).getFileTimeseriesMetdataOffsetMap()); + seriesCompactionExecutor.setType( + compactionSeriesContextMap.get(measurement).getFinalType()); seriesCompactionExecutor.execute(); } } else { diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java index faab7c3d91d,121d4ca1d3a..7ab5769378e --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java @@@ -284,21 -275,22 +287,27 @@@ public class FastAlignedSeriesCompactio } private boolean isValueChunkDataTypeMatchSchema( - List<IChunkMetadata> chunkMetadataListOfOneValueColumn) { + List<IChunkMetadata> chunkMetadataListOfOneValueColumn, + EvolvedSchema evolvedSchema) { + boolean isMatch = false; for (IChunkMetadata chunkMetadata : chunkMetadataListOfOneValueColumn) { if (chunkMetadata == null) { continue; } String measurement = chunkMetadata.getMeasurementUid(); + if (evolvedSchema != null) { + String originalTableName = evolvedSchema.getOriginalTableName(deviceId.getTableName()); + measurement = evolvedSchema.getFinalColumnName(originalTableName, measurement); + } IMeasurementSchema schema = measurementSchemaMap.get(measurement); - return schema.getType() == chunkMetadata.getDataType(); + if (MetadataUtils.canAlter(chunkMetadata.getDataType(), schema.getType())) { + if (schema.getType() != chunkMetadata.getDataType()) { + chunkMetadata.setNewType(schema.getType()); + } + isMatch = true; + } } - return true; + return isMatch; } /** @@@ -379,10 -371,21 +388,26 @@@ valueChunks.add(null); continue; } ++ + Chunk chunk = readChunk(reader, (ChunkMetadata) valueChunkMetadata); + // the column may be renamed, enqueue with the final column name + chunk.getHeader().setMeasurementID(valueChunkMetadata.getMeasurementUid()); - valueChunks.add(chunk); ++ + if (valueChunkMetadata.getNewType() != null) { + Chunk chunk = - readChunk(reader, (ChunkMetadata) valueChunkMetadata) ++ chunk + .rewrite( + ((ChunkMetadata) valueChunkMetadata).getNewType(), chunkMetadataElement.chunk); + valueChunks.add(chunk); + + ChunkMetadata chunkMetadata = (ChunkMetadata) valueChunkMetadata; + chunkMetadata.setTsDataType(valueChunkMetadata.getNewType()); + Statistics<?> statistics = Statistics.getStatsByType(valueChunkMetadata.getNewType()); + statistics.mergeStatistics(chunk.getChunkStatistic()); + chunkMetadata.setStatistics(statistics); + } else { - valueChunks.add(readChunk(reader, (ChunkMetadata) valueChunkMetadata)); ++ valueChunks.add(chunk); + } } chunkMetadataElement.valueChunks = valueChunks; setForceDecoding(chunkMetadataElement); diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java index da9617a7e67,877c94f7081..cf5a23374e6 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java @@@ -65,13 -65,21 +65,21 @@@ public class DeletionPredicate implemen } public boolean matches(IDeviceID deviceID) { - return tableName.equals(deviceID.getTableName()) && idPredicate.matches(deviceID); + return tableName.equals(deviceID.getTableName()) && tagPredicate.matches(deviceID); } - public void setIdPredicate(IDPredicate idPredicate) { - this.idPredicate = idPredicate; + public void setIdPredicate(TagPredicate tagPredicate) { + this.tagPredicate = tagPredicate; } + public IDPredicate getIdPredicate() { + return idPredicate; + } + + public IDPredicate.IDPredicateType getIdPredicateType() { + return this.idPredicate.type; + } + public String getTableName() { return tableName; }
