This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch TableModelIngestion in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f61b76d5687ac88a5e89405315886012d22308af Merge: fdf3f4e7848 0343fc2a777 Author: jt2594838 <[email protected]> AuthorDate: Fri Jun 21 11:38:39 2024 +0800 Merge branch 'ty/TableModelGrammar' into TableModelIngestion # Conflicts: # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java .../java/org/apache/iotdb/CountPointProcessor.java | 14 +- example/schema/pom.xml | 1 - .../apache/iotdb/SubscriptionSessionExample.java | 154 ++- .../iotdb/it/env/cluster/ClusterConstant.java | 3 - .../org/apache/iotdb/it/env/cluster/EnvUtils.java | 6 +- .../it/env/cluster/config/MppCommonConfig.java | 26 + .../env/cluster/config/MppSharedCommonConfig.java | 25 + .../iotdb/it/env/cluster/env/AbstractEnv.java | 4 +- .../it/env/cluster/node/AbstractNodeWrapper.java | 73 +- .../it/env/cluster/node/ConfigNodeWrapper.java | 4 +- .../iotdb/it/env/cluster/node/DataNodeWrapper.java | 4 +- .../it/env/remote/config/RemoteCommonConfig.java | 17 + .../org/apache/iotdb/it/utils/TsFileGenerator.java | 103 +- .../apache/iotdb/itbase/env/BaseNodeWrapper.java | 16 +- .../org/apache/iotdb/itbase/env/CommonConfig.java | 7 + .../iotdb/itbase/runtime/RequestDelegate.java | 10 +- ...TDBLoadTsFileIT.java => IoTDBLoadTsFileIT.java} | 394 +++++--- .../org/apache/iotdb/db/it/IoTDBRecoverIT.java | 2 +- .../iotdb/db/it/IoTDBVerifyConnectionIT.java | 115 +++ ...oTDBAlignByDeviceWithTemplateAggregationIT.java | 130 ++- .../db/it/aligned/IoTDBPredicatePushDownIT.java | 27 + .../regionscan/IoTDBActiveRegionScanIT.java | 12 +- .../regionscan/IoTDBActiveRegionScanIT2.java | 9 +- .../IoTDBActiveRegionScanWithModsIT.java | 26 +- .../IoTDBActiveRegionScanWithModsIT2.java | 9 +- .../regionscan/IoTDBActiveSchemaQueryIT.java | 363 +++++++ .../org/apache/iotdb/db/it/utils/TestUtils.java | 36 + .../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 105 ++ .../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 166 ++++ .../iotdb/pipe/it/single/IoTDBPipeAggregateIT.java | 14 + .../relational/it/schema/IoTDBDatabaseIT.java | 37 +- .../subscription/it/AbstractSubscriptionIT.java | 49 + .../it/IoTDBSubscriptionITConstant.java | 19 +- .../iotdb/subscription/it/SkipOnSetUpFailure.java | 82 ++ .../IoTDBSubscriptionRestartIT.java | 84 +- .../it/dual/AbstractSubscriptionDualIT.java | 13 +- .../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 42 +- .../it/dual/IoTDBSubscriptionTimePrecisionIT.java | 33 +- .../it/dual/IoTDBSubscriptionTopicIT.java | 389 ++++++-- .../it/local/AbstractSubscriptionLocalIT.java | 39 +- .../it/local/IoTDBSubscriptionBasicIT.java | 204 ++-- .../it/local/IoTDBSubscriptionIdempotentIT.java | 14 +- .../apache/iotdb/tools/it/ExportTsFileTestIT.java | 8 +- .../org/apache/iotdb/util/AbstractSchemaIT.java | 8 + .../java/org/apache/iotdb/pipe/api/access/Row.java | 46 +- .../java/org/apache/iotdb/pipe/api/type/Type.java | 15 +- .../org/apache/iotdb/udf/api/utils/RowImpl.java | 8 + .../org/apache/iotdb/tool/AbstractTsFileTool.java | 12 +- .../java/org/apache/iotdb/tool/ExportTsFile.java | 10 +- .../java/org/apache/iotdb/tool/ImportTsFile.java | 4 +- iotdb-client/client-cpp/README.md | 86 +- .../client-cpp/src/assembly/client-cpp.xml | 12 +- .../client-py/iotdb/utils/IoTDBRpcDataSet.py | 14 +- .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 20 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 + .../iotdb/rpc/subscription/config/TopicConfig.java | 48 +- .../rpc/subscription/config/TopicConstant.java | 11 + .../SubscriptionIdentifierSemanticException.java | 30 +- .../payload/poll/SubscriptionPollResponse.java | 3 + .../payload/poll/SubscriptionPollResponseType.java | 2 + .../payload/poll/TerminationPayload.java | 38 +- .../response/PipeSubscribeSubscribeResp.java | 51 +- .../response/PipeSubscribeUnsubscribeResp.java | 51 +- .../consumer/SubscriptionConsumer.java | 116 ++- .../SubscriptionExecutorServiceManager.java | 8 +- .../consumer/SubscriptionProvider.java | 12 +- .../consumer/SubscriptionPullConsumer.java | 9 +- .../consumer/SubscriptionPushConsumer.java | 8 +- .../payload/SubscriptionFileHandler.java | 20 +- .../payload/SubscriptionSessionDataSet.java | 9 +- .../session/subscription/util/IdentifierUtils.java | 51 + .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 13 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 + .../src/assembly/resources/sbin/stop-confignode.sh | 2 + ...RequestType.java => CnToCnNodeRequestType.java} | 4 +- ...NodeRequestType.java => CnToDnRequestType.java} | 5 +- .../async/AsyncConfigNodeHeartbeatClientPool.java | 7 +- .../client/async/AsyncDataNodeClientPool.java | 559 ----------- .../CnToCnInternalServiceAsyncRequestManager.java | 74 ++ .../CnToDnInternalServiceAsyncRequestManager.java | 376 ++++++++ .../handlers/ConfigNodeAsyncRequestContext.java} | 44 +- .../handlers/DataNodeAsyncRequestContext.java | 50 + .../rpc/CheckTimeSeriesExistenceRPCHandler.java | 22 +- .../rpc/ConfigNodeAsyncRequestRPCHandler.java | 88 ++ ...dler.java => ConfigNodeTSStatusRPCHandler.java} | 32 +- .../rpc/CountPathsUsingTemplateRPCHandler.java | 22 +- .../DataNodeAsyncRequestRPCHandler.java} | 175 +--- ...andler.java => DataNodeTSStatusRPCHandler.java} | 12 +- .../rpc/FetchSchemaBlackListRPCHandler.java | 20 +- .../handlers/rpc/PipeHeartbeatRPCHandler.java | 8 +- .../async/handlers/rpc/PipePushMetaRPCHandler.java | 8 +- .../async/handlers/rpc/SchemaUpdateRPCHandler.java | 14 +- ...ava => SubmitTestConnectionTaskRPCHandler.java} | 42 +- ...itTestConnectionTaskToConfigNodeRPCHandler.java | 74 ++ .../handlers/rpc/TransferLeaderRPCHandler.java | 9 +- .../CheckSchemaRegionUsingTemplateRPCHandler.java | 27 +- .../ConsumerGroupPushMetaRPCHandler.java | 10 +- .../rpc/subscription/TopicPushMetaRPCHandler.java | 10 +- .../client/sync/SyncConfigNodeClientPool.java | 12 +- .../client/sync/SyncDataNodeClientPool.java | 9 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 15 + .../confignode/conf/ConfigNodeRemoveCheck.java | 4 +- .../confignode/conf/ConfigNodeStartupCheck.java | 2 +- .../conf/ConfigNodeSystemPropertiesHandler.java} | 36 +- .../confignode/conf/SystemPropertiesUtils.java | 86 +- .../consensus/request/read/ttl/ShowTTLPlan.java | 13 + .../response/datanode/DataNodeRegisterResp.java | 6 +- .../iotdb/confignode/manager/ClusterManager.java | 221 +++++ .../confignode/manager/ClusterQuotaManager.java | 23 +- .../iotdb/confignode/manager/ConfigManager.java | 20 +- .../apache/iotdb/confignode/manager/IManager.java | 2 +- .../iotdb/confignode/manager/TTLManager.java | 4 +- .../iotdb/confignode/manager/TriggerManager.java | 14 +- .../iotdb/confignode/manager/UDFManager.java | 20 +- .../confignode/manager/cq/CQScheduleTask.java | 5 +- .../manager/load/balancer/RouteBalancer.java | 26 +- .../manager/node/ClusterNodeStartUtils.java | 121 ++- .../iotdb/confignode/manager/node/NodeManager.java | 90 +- .../manager/partition/PartitionManager.java | 39 +- .../pipe/agent/task/PipeConfigNodeTaskAgent.java | 10 + .../protocol/IoTDBConfigRegionAirGapConnector.java | 7 +- .../protocol/IoTDBConfigRegionConnector.java | 7 +- .../runtime/heartbeat/PipeHeartbeatScheduler.java | 15 +- .../pipe/event/PipeConfigRegionSnapshotEvent.java | 13 +- .../pipe/event/PipeConfigRegionWritePlanEvent.java | 8 +- .../execution/PipeConfigNodeSubtaskExecutor.java | 6 +- .../metric/PipeConfigNodeRemainingTimeMetrics.java | 32 +- .../PipeConfigNodeRemainingTimeOperator.java | 99 +- .../metric/PipeConfigRegionConnectorMetrics.java | 2 +- .../manager/pipe/task/PipeConfigNodeTaskStage.java | 16 +- .../manager/schema/ClusterSchemaManager.java | 14 +- .../confignode/persistence/ProcedureInfo.java | 2 +- .../iotdb/confignode/persistence/TTLInfo.java | 21 +- .../persistence/executor/ConfigPlanExecutor.java | 3 +- .../schema/CNPhysicalPlanGenerator.java | 11 +- .../procedure/env/ConfigNodeProcedureEnv.java | 219 ++--- .../procedure/env/RegionMaintainHandler.java | 31 +- .../impl/schema/AlterLogicalViewProcedure.java | 18 +- .../impl/schema/DataNodeRegionTaskExecutor.java | 25 +- .../impl/schema/DeactivateTemplateProcedure.java | 27 +- .../impl/schema/DeleteDatabaseProcedure.java | 16 +- .../impl/schema/DeleteLogicalViewProcedure.java | 22 +- .../impl/schema/DeleteTimeSeriesProcedure.java | 26 +- .../procedure/impl/schema/SchemaUtils.java | 6 +- .../procedure/impl/schema/SetTTLProcedure.java | 14 +- .../impl/schema/SetTemplateProcedure.java | 30 +- .../impl/schema/UnsetTemplateProcedure.java | 22 +- .../impl/schema/table/AddTableColumnProcedure.java | 24 +- .../impl/schema/table/CreateTableProcedure.java | 26 +- .../subscription/CreateSubscriptionProcedure.java | 2 + .../subscription/DropSubscriptionProcedure.java | 151 +-- .../impl/sync/AuthOperationProcedure.java | 4 +- .../iotdb/confignode/service/ConfigNode.java | 8 +- .../confignode/service/ConfigNodeShutdownHook.java | 4 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 28 +- .../iotdb/confignode/persistence/NodeInfoTest.java | 3 +- .../iotdb/confignode/persistence/TTLInfoTest.java | 39 +- .../apache/iotdb/consensus/ConsensusFactory.java | 9 +- .../consensus/pipe/PipeConsensusServerImpl.java | 67 +- .../consensuspipe/ConsensusPipeConnector.java} | 16 +- .../pipe/consensuspipe/ConsensusPipeManager.java | 4 + .../pipe/metric/PipeConsensusServerMetrics.java | 190 ++++ .../pipe/metric/PipeConsensusSyncLagManager.java | 131 +++ .../iotdb/consensus/ratis/RatisConsensus.java | 22 +- .../src/assembly/resources/sbin/stop-datanode.sh | 3 + .../org/apache/iotdb/db/auth/AuthorityChecker.java | 32 +- .../DataNodeSystemPropertiesHandler.java} | 38 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 63 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 71 +- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 173 +--- .../db/consensus/DataRegionConsensusImpl.java | 9 +- .../schemaregion/SchemaExecutionVisitor.java | 21 +- .../schemaregion/SchemaRegionStateMachine.java | 16 +- .../{PipeAgent.java => PipeDataNodeAgent.java} | 20 +- .../db/pipe/agent/runtime/PipeAgentLauncher.java | 6 +- .../agent/runtime/PipeDataNodeRuntimeAgent.java | 10 +- .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 43 +- .../IoTDBDataNodeCacheLeaderClientManager.java | 21 +- .../payload/evolvable/builder/PipeEventBatch.java | 18 +- .../request/PipeTransferTabletRawReq.java | 70 +- .../airgap/IoTDBDataNodeAirGapConnector.java | 1 + .../airgap/IoTDBDataRegionAirGapConnector.java | 16 +- .../airgap/IoTDBSchemaRegionAirGapConnector.java | 6 +- .../connector/protocol/opcua/OpcUaConnector.java | 14 + .../pipeconsensus/PipeConsensusAsyncConnector.java | 151 ++- .../pipeconsensus/PipeConsensusSyncConnector.java | 25 +- .../PipeConsensusTabletBatchEventHandler.java | 7 +- .../PipeConsensusTabletInsertNodeEventHandler.java | 6 +- .../PipeConsensusTabletInsertionEventHandler.java | 14 +- .../PipeConsensusTabletRawEventHandler.java | 45 - .../PipeConsensusTsFileInsertionEventHandler.java | 22 +- .../PipeTransferTabletBatchEventHandler.java | 9 +- .../PipeTransferTabletInsertionEventHandler.java | 22 +- .../PipeTransferTsFileInsertionEventHandler.java | 8 +- .../thrift/sync/IoTDBDataNodeSyncConnector.java | 1 + .../thrift/sync/IoTDBDataRegionSyncConnector.java | 20 +- .../thrift/sync/IoTDBSchemaRegionConnector.java | 6 +- .../protocol/writeback/WriteBackConnector.java | 4 +- .../consensus/ConsensusPipeDataNodeDispatcher.java | 4 +- .../ConsensusPipeDataNodeRuntimeAgentGuardian.java | 4 +- .../consensus/PipeConsensusConnectorMetrics.java | 290 ++++++ .../consensus/PipeConsensusReceiverMetrics.java | 384 ++++++++ .../consensus/PipeConsensusSubtaskExecutor.java | 4 +- .../consensus/ProgressIndexDataNodeManager.java | 6 +- .../db/pipe/event/UserDefinedEnrichedEvent.java | 28 +- .../event/common/heartbeat/PipeHeartbeatEvent.java | 20 +- .../event/common/row/PipeDataTypeTransformer.java | 14 +- .../iotdb/db/pipe/event/common/row/PipeRow.java | 53 +- .../db/pipe/event/common/row/PipeRowCollector.java | 1 + .../schema/PipeSchemaRegionSnapshotEvent.java | 14 +- .../schema/PipeSchemaRegionWritePlanEvent.java | 8 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 150 +-- .../common/tablet/PipeRawTabletInsertionEvent.java | 83 +- .../tablet/TabletInsertionDataContainer.java | 99 +- .../event/common/terminate/PipeTerminateEvent.java | 19 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 106 +- .../tsfile/TsFileInsertionDataContainer.java | 73 +- .../db/pipe/event/realtime/PipeRealtimeEvent.java | 40 +- .../execution/PipeConnectorSubtaskExecutor.java | 7 +- .../execution/PipeProcessorSubtaskExecutor.java | 3 +- .../PipeHistoricalDataRegionTsFileExtractor.java | 110 ++- .../realtime/PipeRealtimeDataRegionExtractor.java | 86 +- .../PipeRealtimeDataRegionHybridExtractor.java | 8 +- .../PipeRealtimeDataRegionLogExtractor.java | 11 +- .../PipeRealtimeDataRegionTsFileExtractor.java | 8 +- .../realtime/assigner/PipeDataRegionAssigner.java | 2 +- .../listener/PipeInsertionDataNodeListener.java | 4 +- .../schemaregion/IoTDBSchemaRegionExtractor.java | 13 +- .../PipeDataNodeRemainingEventAndTimeMetrics.java | 58 +- .../PipeDataNodeRemainingEventAndTimeOperator.java | 178 ++-- .../metric/PipeDataRegionConnectorMetrics.java | 4 +- .../db/pipe/metric/PipeDataRegionEventCounter.java | 7 + .../metric/PipeDataRegionExtractorMetrics.java | 8 +- .../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 48 +- .../metric/PipeSchemaRegionConnectorMetrics.java | 2 +- .../processor/aggregate/AggregateProcessor.java | 47 +- .../aggregate/TimeSeriesRuntimeState.java | 94 ++ .../CustomizedReadableIntermediateResults.java | 49 +- .../IntermediateResultOperator.java | 41 +- .../numeric/AbstractSameTypeNumericOperator.java | 46 +- .../doubletype/FractionPoweredSumOperator.java | 56 +- .../specifictype/integertype/CountOperator.java | 54 +- .../window/datastructure/TimeSeriesWindow.java | 111 +++ .../AbstractSimpleTimeWindowingProcessor.java | 52 +- .../processor/AbstractWindowingProcessor.java | 45 +- .../downsampling/PartialPathLastObjectCache.java | 16 +- .../downsampling/changing/ChangingValueFilter.java | 22 +- .../sdt/SwingingDoorTrendingFilter.java | 20 +- .../twostage/combiner/PipeCombineHandler.java | 4 +- .../combiner/PipeCombineHandlerManager.java | 4 +- .../twostage/plugin/TwoStageCountProcessor.java | 2 +- .../protocol/airgap/IoTDBAirGapReceiver.java | 6 +- .../pipeconsensus/PipeConsensusReceiver.java | 470 ++++++--- .../protocol/thrift/IoTDBDataNodeReceiver.java | 4 +- .../visitor/PipeStatementExceptionVisitor.java | 2 +- .../visitor/PipeStatementTSStatusVisitor.java | 33 +- .../db/pipe/resource/memory/PipeMemoryManager.java | 4 +- .../resource/tsfile/PipeTsFileResourceManager.java | 42 +- .../pipe/resource/wal/PipeWALResourceManager.java | 4 +- .../pipe/task/builder/PipeDataNodeTaskBuilder.java | 4 +- .../pipe/task/connection/PipeEventCollector.java | 14 +- .../db/pipe/task/stage/PipeTaskExtractorStage.java | 6 +- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 6 +- .../subtask/connector/PipeConnectorSubtask.java | 30 +- .../connector/PipeConnectorSubtaskManager.java | 8 +- .../subtask/processor/PipeProcessorSubtask.java | 22 +- .../processor/PipeProcessorSubtaskWorker.java | 21 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 34 +- .../iotdb/db/protocol/client/ConfigNodeInfo.java | 66 +- .../cn/AsyncConfigNodeTSStatusRPCHandler.java} | 34 +- .../cn/ConfigNodeAsyncRequestRPCHandler.java | 79 ++ .../DnToCnInternalServiceAsyncRequestManager.java | 64 ++ .../db/protocol/client/cn/DnToCnRequestType.java} | 17 +- .../client/dn}/AsyncTSStatusRPCHandler.java | 9 +- .../client/dn/DataNodeAsyncRequestRPCHandler.java | 84 ++ ...DataNodeExternalServiceAsyncRequestManager.java | 83 ++ .../dn/DataNodeMPPServiceAsyncRequestManager.java | 84 ++ .../DnToDnInternalServiceAsyncRequestManager.java | 64 ++ .../db/protocol/client/dn/DnToDnRequestType.java} | 16 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 34 +- .../impl/DataNodeInternalRPCServiceImpl.java | 125 ++- .../thrift/impl/DataNodeRegionManager.java | 4 +- .../iotdb/db/queryengine/common/DeviceContext.java | 81 ++ ...eriesSchemaInfo.java => TimeseriesContext.java} | 48 +- .../common/header/ColumnHeaderConstant.java | 11 + .../common/header/DatasetHeaderFactory.java | 4 + .../common/schematree/ClusterSchemaTree.java | 13 +- .../common/schematree/IMeasurementSchemaInfo.java | 2 + .../common/schematree/MeasurementSchemaInfo.java | 13 +- .../schematree/node/SchemaMeasurementNode.java | 14 +- .../visitor/SchemaTreeDeviceVisitor.java | 3 +- .../execution/driver/DataDriverContext.java | 19 +- .../execution/exchange/MPPDataExchangeManager.java | 22 +- .../fragment/FragmentInstanceContext.java | 29 +- .../fragment/FragmentInstanceExecution.java | 2 +- .../fragment/FragmentInstanceManager.java | 10 + .../execution/load/AlignedChunkData.java | 124 ++- .../execution/load/LoadTsFileManager.java | 147 ++- .../execution/load/LoadTsFileRateLimiter.java | 45 +- .../execution/load/NonAlignedChunkData.java | 65 +- .../queryengine/execution/memory/MemoryPool.java | 9 +- .../operator/schema/SchemaFetchScanOperator.java | 73 +- .../operator/schema/source/DeviceSchemaSource.java | 4 +- ...erator.java => AbstractSeriesScanOperator.java} | 123 +-- .../source/ActiveDeviceRegionScanOperator.java | 43 +- .../source/ActiveTimeSeriesRegionScanOperator.java | 14 +- .../operator/source/AlignedSeriesScanOperator.java | 110 +-- .../source/RegionScanForActiveDeviceUtil.java | 3 +- .../source/RegionScanForActiveTimeSeriesUtil.java | 6 +- .../operator/source/SeriesScanOperator.java | 111 +-- .../load/LoadTsFileAnalyzeSchemaMemoryBlock.java | 2 +- .../queryengine/load/LoadTsFileMemoryManager.java | 10 +- .../metric/load/LoadTsFileCostMetricsSet.java | 131 +++ .../metric/{ => load}/LoadTsFileMemMetricSet.java | 2 +- .../db/queryengine/plan/analyze/Analysis.java | 19 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 169 +++- .../plan/analyze/ClusterPartitionFetcher.java | 19 + .../db/queryengine/plan/analyze/IAnalysis.java | 2 + .../plan/analyze/IPartitionFetcher.java | 32 + .../plan/analyze/LoadTsfileAnalyzer.java | 40 +- .../plan/analyze/TemplatedAggregationAnalyze.java | 58 +- .../queryengine/plan/analyze/TemplatedAnalyze.java | 11 +- .../analyze/cache/schema/DataNodeTTLCache.java | 20 - .../schema/DeviceUsingTemplateSchemaCache.java | 5 + .../analyze/cache/schema/SchemaCacheEntry.java | 5 + .../cache/schema/TimeSeriesSchemaCache.java | 2 + .../analyze/schema/AutoCreateSchemaExecutor.java | 2 + .../analyze/schema/ClusterSchemaFetchExecutor.java | 46 +- .../plan/analyze/schema/ClusterSchemaFetcher.java | 16 + .../plan/analyze/schema/ISchemaFetcher.java | 25 +- .../queryengine/plan/execution/QueryExecution.java | 3 +- .../plan/execution/config/ConfigTaskVisitor.java | 8 + .../config/executor/ClusterConfigTaskExecutor.java | 44 +- .../config/executor/IConfigTaskExecutor.java | 2 + .../execution/config/sys/TestConnectionTask.java | 218 +++++ .../plan/expression/multi/FunctionExpression.java | 8 + .../db/queryengine/plan/parser/ASTVisitor.java | 30 +- .../plan/parser/StatementGenerator.java | 2 +- .../plan/planner/LocalExecutionPlanner.java | 28 +- .../plan/planner/LogicalPlanBuilder.java | 52 +- .../plan/planner/LogicalPlanVisitor.java | 38 +- .../plan/planner/OperatorTreeGenerator.java | 168 +++- .../planner/distribution/ExchangeNodeAdder.java | 12 +- .../plan/planner/plan/node/PlanNodeType.java | 10 +- .../plan/planner/plan/node/PlanVisitor.java | 14 +- .../metedata/read/DeviceSchemaFetchScanNode.java | 101 ++ .../node/metedata/read/SchemaFetchScanNode.java | 123 +-- ...canNode.java => SeriesSchemaFetchScanNode.java} | 115 +-- .../write/CreateAlignedTimeSeriesNode.java | 3 +- .../plan/node/source/DeviceRegionScanNode.java | 46 +- .../plan/node/source/TimeseriesRegionScanNode.java | 37 +- .../planner/plan/node/write/InsertRowNode.java | 8 +- .../plan/node/write/InsertRowsOfOneDeviceNode.java | 5 +- .../planner/plan/node/write/InsertTabletNode.java | 8 +- .../plan/relational/analyzer/Analysis.java | 11 +- .../relational/analyzer/StatementAnalyzer.java | 9 + .../plan/relational/metadata/Metadata.java | 47 +- .../plan/relational/metadata/MetadataUtil.java | 2 +- .../relational/metadata/TableMetadataImpl.java | 24 + .../plan/relational/planner/LogicalPlanner.java | 30 + .../planner/node/CreateTableDeviceNode.java | 316 ++++++ .../planner/optimizations/IndexScan.java | 8 +- .../plan/relational/sql/ast/AstVisitor.java | 8 + .../plan/relational/sql/ast/CreateDevice.java | 116 +++ .../plan/relational/sql/ast/GroupByTime.java | 145 +++ .../plan/relational/sql/ast/SimpleGroupBy.java | 2 +- .../sql/ast/{SimpleGroupBy.java => TimeRange.java} | 74 +- .../plan/relational/sql/parser/AstBuilder.java | 182 ++-- .../plan/relational/sql/parser/SqlParser.java | 52 +- .../relational/sql/util/ReservedIdentifiers.java | 3 +- .../scheduler/load/LoadTsFileDispatcherImpl.java | 6 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 46 +- .../plan/statement/StatementVisitor.java | 18 +- .../plan/statement/crud/InsertTabletStatement.java | 4 + ...tement.java => DeviceSchemaFetchStatement.java} | 37 +- ...tement.java => SeriesSchemaFetchStatement.java} | 22 +- .../statement/metadata/CountDevicesStatement.java | 27 + .../metadata/CountTimeSeriesStatement.java | 27 + .../statement/metadata/ShowDevicesStatement.java | 27 + .../plan/statement/metadata/ShowTTLStatement.java | 11 +- .../metadata/ShowTimeSeriesStatement.java | 27 + .../TestConnectionStatement.java} | 32 +- .../schemaengine/schemaregion/ISchemaRegion.java | 22 +- .../attribute/DeviceAttributeStore.java | 19 +- .../attribute/IDeviceAttributeStore.java | 4 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 29 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 26 +- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 87 +- .../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 62 +- .../schemaengine/schemaregion/tag/TagManager.java | 15 + .../java/org/apache/iotdb/db/service/DataNode.java | 60 +- .../db/service/metrics/DataNodeMetricsHelper.java | 6 +- .../iotdb/db/storageengine/StorageEngine.java | 8 +- .../db/storageengine/dataregion/DataRegion.java | 7 +- .../dataregion/IDataRegionForQuery.java | 3 +- .../dataregion/VirtualDataRegion.java | 3 +- .../CompactionLastTimeCheckFailedException.java | 17 + .../CompactionValidationFailedException.java | 14 +- .../impl/ReadChunkCompactionPerformer.java | 7 +- .../execute/task/AbstractCompactionTask.java | 17 +- .../execute/utils/CompactionPathUtils.java | 13 +- .../execute/utils/MultiTsFileDeviceIterator.java | 120 +-- .../readchunk/SingleSeriesCompactionExecutor.java | 20 +- .../executor/readchunk/loader/ChunkLoader.java | 3 + .../writer/RepairUnsortedFileCompactionWriter.java | 18 +- .../dataregion/memtable/TsFileProcessor.java | 15 +- .../allocation/AbstractNodeAllocationStrategy.java | 8 +- .../dataregion/wal/buffer/AbstractWALBuffer.java | 3 +- .../dataregion/wal/buffer/WALBuffer.java | 16 +- .../wal/checkpoint/CheckpointManager.java | 8 +- .../dataregion/wal/io/CheckpointReader.java | 5 +- .../dataregion/wal/io/CheckpointWriter.java | 4 +- .../dataregion/wal/io/ILogWriter.java | 3 +- .../storageengine/dataregion/wal/io/LogWriter.java | 88 +- .../dataregion/wal/io/WALByteBufReader.java | 12 +- .../dataregion/wal/io/WALInputStream.java | 365 +++++++ .../dataregion/wal/io/WALMetaData.java | 25 +- .../storageengine/dataregion/wal/io/WALReader.java | 12 +- .../storageengine/dataregion/wal/io/WALWriter.java | 30 +- .../storageengine/dataregion/wal/node/WALNode.java | 5 +- .../dataregion/wal/recover/WALNodeRecoverTask.java | 1 + .../dataregion/wal/recover/WALRecoverWriter.java | 4 +- .../file/AbstractTsFileRecoverPerformer.java | 4 +- .../file/UnsealedTsFileRecoverPerformer.java | 4 +- .../dataregion/wal/utils/WALEntryPosition.java | 31 +- .../dataregion/wal/utils/WALInsertNodeCache.java | 43 +- .../storageengine/rescon/disk/FolderManager.java | 4 + .../agent/SubscriptionAgentLauncher.java | 8 +- .../agent/SubscriptionBrokerAgent.java | 11 +- .../agent/SubscriptionConsumerAgent.java | 29 +- .../agent/SubscriptionReceiverAgent.java | 6 +- .../subscription/agent/SubscriptionTopicAgent.java | 32 +- .../db/subscription/broker/SubscriptionBroker.java | 46 +- .../broker/SubscriptionPrefetchingQueue.java | 60 +- .../SubscriptionPrefetchingTabletsQueue.java | 21 + .../broker/SubscriptionPrefetchingTsFileQueue.java | 59 +- .../db/subscription/event/SubscriptionEvent.java | 6 + .../event/SubscriptionEventBinaryCache.java | 8 +- .../receiver/SubscriptionReceiverV1.java | 37 +- .../task/subtask/SubscriptionConnectorSubtask.java | 18 +- .../SubscriptionConnectorSubtaskLifeCycle.java | 57 +- .../SubscriptionConnectorSubtaskManager.java | 4 +- .../java/org/apache/iotdb/db/tools/WalChecker.java | 6 +- .../org/apache/iotdb/db/utils/DateTimeUtils.java | 79 +- .../java/org/apache/iotdb/db/utils/MemUtils.java | 2 +- .../apache/iotdb/db/utils/TypeInferenceUtils.java | 21 +- .../db/metadata/cache/DataNodeSchemaCacheTest.java | 6 + .../schemaRegion/SchemaRegionBasicTest.java | 13 +- .../SchemaRegionSimpleRecoverTest.java | 18 +- .../schemaRegion/SchemaRegionTemplateTest.java | 19 +- .../schemaRegion/SchemaRegionTestUtil.java | 7 +- .../connector/PipeDataNodeThriftRequestTest.java | 116 ++- .../db/pipe/event/PipeDataTypeTransformerTest.java | 13 +- .../pipe/event/PipeTabletInsertionEventTest.java | 40 +- .../extractor/SchemaRegionListeningQueueTest.java | 22 +- .../pattern/CachedSchemaPatternMatcherTest.java | 30 +- .../resource/PipeTsFileResourceManagerTest.java | 8 +- .../common/schematree/ClusterSchemaTreeTest.java | 2 + .../execution/operator/OperatorMemoryTest.java | 4 +- .../schema/SchemaFetchScanOperatorTest.java | 16 +- .../plan/analyze/FakePartitionFetcherImpl.java | 18 + .../plan/analyze/FakeSchemaFetcherImpl.java | 14 + .../plan/analyze/QueryTimePartitionTest.java | 33 +- .../plan/planner/distribution/Util.java | 29 + .../plan/planner/distribution/Util2.java | 29 + .../logical/RegionScanLogicalPlannerTest.java | 94 +- ...est.java => DeviceSchemaFetchScanNodeTest.java} | 26 +- .../metadata/read/SchemaFetchMergeNodeTest.java | 12 +- ...est.java => SeriesSchemaFetchScanNodeTest.java} | 17 +- .../plan/relational/analyzer/AnalyzerTest.java | 19 +- .../plan/relational/analyzer/TestMatadata.java | 18 + .../iotdb/db/storageengine/dataregion/TTLTest.java | 3 +- .../repair/RepairUnsortedFileCompactionTest.java | 26 + .../storageengine/dataregion/wal/WALTestUtils.java | 90 ++ .../wal/compression/WALCompressionTest.java | 409 ++++++++ .../dataregion/wal/node/WALNodeTest.java | 4 + .../wal/recover/WALRecoverManagerTest.java | 4 + .../wal/recover/WALRecoverWriterTest.java | 14 +- .../metrics/metricsets/system/SystemMetrics.java | 4 +- iotdb-core/node-commons/pom.xml | 8 + .../resources/conf/generate_properties.bat | 3 +- .../resources/conf/iotdb-system.properties | 44 +- .../src/assembly/resources/sbin/iotdb-common.sh | 23 + .../iotdb/commons/client/ClientPoolFactory.java | 58 +- ...a => AsyncConfigNodeInternalServiceClient.java} | 23 +- ...ava => AsyncDataNodeExternalServiceClient.java} | 39 +- .../container/PipeConsensusClientMgrContainer.java | 2 +- .../client/request/AsyncRequestContext.java | 138 +++ .../client/request/AsyncRequestManager.java | 192 ++++ .../client/request/AsyncRequestRPCHandler.java} | 49 +- ...nfigNodeInternalServiceAsyncRequestManager.java | 43 + .../DataNodeInternalServiceRequestManager.java | 43 + .../apache/iotdb/commons/client/request/Utils.java | 78 ++ .../threadpool/WrappedThreadPoolExecutor.java | 7 +- .../apache/iotdb/commons/conf/CommonConfig.java | 35 +- .../iotdb/commons/conf/CommonDescriptor.java | 21 +- .../iotdb/commons/conf/ConfigurationFileUtils.java | 68 +- .../enums/PipeRemainingTimeRateAverageTime.java} | 34 +- .../iotdb/commons/file/SystemFileFactory.java | 4 - .../commons/file/SystemPropertiesHandler.java | 180 ++++ .../iotdb/commons/partition/SchemaPartition.java | 15 + .../org/apache/iotdb/commons/path/PartialPath.java | 13 + .../iotdb/commons/path/PathPatternTreeUtils.java | 5 + .../commons/pipe/agent/task/PipeTaskAgent.java | 45 +- .../iotdb/commons/pipe/config/PipeConfig.java | 22 +- .../config/constant/PipeConnectorConstant.java | 1 + .../config/constant/PipeExtractorConstant.java | 6 + .../pipe/connector/compressor/PipeCompressor.java | 19 + .../connector/compressor/PipeGZIPCompressor.java | 7 + .../connector/compressor/PipeLZ4Compressor.java | 7 + .../connector/compressor/PipeLZMA2Compressor.java | 7 + .../connector/compressor/PipeSnappyCompressor.java | 7 + .../connector/compressor/PipeZSTDCompressor.java | 5 + .../pipe/connector/limiter/GlobalRateLimiter.java | 43 +- .../connector/limiter/PipeEndPointRateLimiter.java | 39 +- .../thrift/request/PipeTransferCompressedReq.java | 36 +- .../connector/protocol/IoTDBAirGapConnector.java | 38 +- .../pipe/connector/protocol/IoTDBConnector.java | 15 +- .../connector/protocol/IoTDBSslSyncConnector.java | 14 +- .../queue/ConcurrentIterableLinkedQueue.java | 27 +- .../iotdb/commons/pipe/event/EnrichedEvent.java | 38 +- .../commons/pipe/event/PipeSnapshotEvent.java | 8 +- .../commons/pipe/event/PipeWritePlanEvent.java | 8 +- .../commons/pipe/event/ProgressReportEvent.java | 12 +- .../execution/executor/PipeSubtaskExecutor.java | 30 +- .../extractor/IoTDBNonDataRegionExtractor.java | 11 +- .../commons/pipe/metric/PipeRemainingOperator.java | 87 ++ .../commons/pipe/pattern/IoTDBPipePattern.java | 4 +- .../pipe/progress/PipeEventCommitManager.java | 51 +- .../commons/pipe/progress/PipeEventCommitter.java | 4 + .../commons/pipe/task/meta/PipeTemporaryMeta.java | 8 +- .../task/subtask/PipeAbstractConnectorSubtask.java | 42 +- .../pipe/task/subtask/PipeReportableSubtask.java | 28 +- .../apache/iotdb/commons/schema/ttl/TTLCache.java | 60 +- .../iotdb/commons/service/metric/enums/Metric.java | 11 +- .../meta/consumer/ConsumerGroupMeta.java | 79 +- .../commons/subscription/meta/topic/TopicMeta.java | 4 + .../iotdb/commons/utils/TimePartitionUtils.java | 4 + .../utils/function/CheckedTriConsumer.java} | 16 +- .../db/relational/grammar/sql/RelationalSql.g4 | 13 +- .../thrift-commons/src/main/thrift/common.thrift | 45 +- .../src/main/thrift/confignode.thrift | 13 +- .../thrift-datanode/src/main/thrift/client.thrift | 3 + .../src/main/thrift/datanode.thrift | 8 + pom.xml | 1010 +------------------- 546 files changed, 15878 insertions(+), 6904 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 533a50d8482,b94d0d75e0a..3af6468e4b9 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@@ -186,8 -191,7 +190,9 @@@ import static org.apache.iotdb.db.query import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.ENDTIME; import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.PARTITION_FETCHER; import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.SCHEMA_FETCHER; +import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeUtils.removeLogicalView; +import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeUtils.validateSchema; + import static org.apache.iotdb.db.queryengine.metric.load.LoadTsFileCostMetricsSet.ANALYSIS; import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.bindSchemaForExpression; import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.concatDeviceAndBindSchemaForExpression; import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.getMeasurementExpression; @@@ -2774,14 -2830,73 +2797,26 @@@ public class AnalyzeVisitor extends Sta return analysis; } - private void validateSchema( - Analysis analysis, InsertBaseStatement insertStatement, MPPQueryContext context) { - final long startTime = System.nanoTime(); - try { - SchemaValidator.validate(schemaFetcher, insertStatement, context); - } catch (SemanticException e) { - analysis.setFinishQueryAfterAnalyze(true); - if (e.getCause() instanceof IoTDBException) { - IoTDBException exception = (IoTDBException) e.getCause(); - analysis.setFailStatus( - RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage())); - } else { - analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage())); - } - } finally { - PERFORMANCE_OVERVIEW_METRICS.recordScheduleSchemaValidateCost(System.nanoTime() - startTime); - } - boolean hasFailedMeasurement = insertStatement.hasFailedMeasurements(); - String partialInsertMessage; - if (hasFailedMeasurement) { - partialInsertMessage = - String.format( - "Fail to insert measurements %s caused by %s", - insertStatement.getFailedMeasurements(), insertStatement.getFailedMessages()); - logger.warn(partialInsertMessage); - analysis.setFailStatus( - RpcUtils.getStatus(TSStatusCode.METADATA_ERROR.getStatusCode(), partialInsertMessage)); - } - } - - private InsertBaseStatement removeLogicalView( - Analysis analysis, InsertBaseStatement insertBaseStatement) { - try { - return insertBaseStatement.removeLogicalView(); - } catch (SemanticException e) { - analysis.setFinishQueryAfterAnalyze(true); - if (e.getCause() instanceof IoTDBException) { - IoTDBException exception = (IoTDBException) e.getCause(); - analysis.setFailStatus( - RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage())); - } else { - analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage())); - } - return insertBaseStatement; - } - } - @Override public Analysis visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) { - LoadTsfileAnalyzer loadTsfileAnalyzer = - new LoadTsfileAnalyzer(loadTsFileStatement, context, partitionFetcher, schemaFetcher); - try { + long startTime = System.nanoTime(); + try (final LoadTsfileAnalyzer loadTsfileAnalyzer = + new LoadTsfileAnalyzer(loadTsFileStatement, context, partitionFetcher, schemaFetcher)) { return loadTsfileAnalyzer.analyzeFileByFile(); + } catch (final Exception e) { + final String exceptionMessage = + String.format( + "Failed to execute load tsfile statement %s. Detail: %s", + loadTsFileStatement, + e.getMessage() == null ? e.getClass().getName() : e.getMessage()); + logger.warn(exceptionMessage, e); + final Analysis analysis = new Analysis(); + analysis.setFinishQueryAfterAnalyze(true); + analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exceptionMessage)); + return analysis; } finally { - loadTsfileAnalyzer.close(); + LoadTsFileCostMetricsSet.getInstance() + .recordPhaseTimeCost(ANALYSIS, System.nanoTime() - startTime); } } @@@ -2977,16 -3124,19 +3026,19 @@@ public Analysis visitShowDevices( ShowDevicesStatement showDevicesStatement, MPPQueryContext context) { Analysis analysis = new Analysis(); - analysis.setStatement(showDevicesStatement); + analysis.setRealStatement(showDevicesStatement); - PathPatternTree patternTree = new PathPatternTree(); - patternTree.appendPathPattern( - showDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); - if (showDevicesStatement.hasTimeCondition()) { analyzeDeviceRegionScan( - showDevicesStatement.getTimeCondition(), patternTree, analysis, context); + showDevicesStatement.getTimeCondition(), + showDevicesStatement.getPathPattern(), + showDevicesStatement.getAuthorityScope(), + analysis, + context); } else { + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern( + showDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); SchemaPartition schemaPartitionInfo = partitionFetcher.getSchemaPartition(patternTree); analysis.setSchemaPartitionInfo(schemaPartitionInfo); } @@@ -3020,13 -3170,34 +3072,34 @@@ } @Override - public Analysis visitSchemaFetch( - SchemaFetchStatement schemaFetchStatement, MPPQueryContext context) { + public Analysis visitSeriesSchemaFetch( + SeriesSchemaFetchStatement seriesSchemaFetchStatement, MPPQueryContext context) { Analysis analysis = new Analysis(); - analysis.setRealStatement(schemaFetchStatement); - analysis.setStatement(seriesSchemaFetchStatement); ++ analysis.setRealStatement(seriesSchemaFetchStatement); SchemaPartition schemaPartition = - partitionFetcher.getSchemaPartition(schemaFetchStatement.getPatternTree()); + partitionFetcher.getSchemaPartition(seriesSchemaFetchStatement.getPatternTree()); + analysis.setSchemaPartitionInfo(schemaPartition); + + if (schemaPartition.isEmpty()) { + analysis.setFinishQueryAfterAnalyze(true); + } + + return analysis; + } + + @Override + public Analysis visitDeviceSchemaFetch( + DeviceSchemaFetchStatement deviceSchemaFetchStatement, MPPQueryContext context) { + Analysis analysis = new Analysis(); - analysis.setStatement(deviceSchemaFetchStatement); ++ analysis.setRealStatement(deviceSchemaFetchStatement); + + PathPatternTree patternTree = new PathPatternTree(); + for (PartialPath path : deviceSchemaFetchStatement.getPaths()) { + patternTree.appendPathPattern(path); + } + patternTree.constructTree(); + SchemaPartition schemaPartition = partitionFetcher.getSchemaPartition(patternTree); analysis.setSchemaPartitionInfo(schemaPartition); if (schemaPartition.isEmpty()) { @@@ -3040,15 -3211,19 +3113,19 @@@ public Analysis visitCountDevices( CountDevicesStatement countDevicesStatement, MPPQueryContext context) { Analysis analysis = new Analysis(); - analysis.setStatement(countDevicesStatement); + analysis.setRealStatement(countDevicesStatement); - PathPatternTree patternTree = new PathPatternTree(); - patternTree.appendPathPattern( - countDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); if (countDevicesStatement.hasTimeCondition()) { analyzeDeviceRegionScan( - countDevicesStatement.getTimeCondition(), patternTree, analysis, context); + countDevicesStatement.getTimeCondition(), + countDevicesStatement.getPathPattern(), + countDevicesStatement.getAuthorityScope(), + analysis, + context); } else { + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern( + countDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); SchemaPartition schemaPartitionInfo = partitionFetcher.getSchemaPartition(patternTree); analysis.setSchemaPartitionInfo(schemaPartitionInfo); } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java index 23a04679e64,70cff5c0779..ce9f043c9bb --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java @@@ -50,18 -49,10 +50,20 @@@ public interface IAnalysis String getStatementType(); + void setFinishQueryAfterAnalyze(boolean b); + + void setFailStatus(TSStatus status); + + boolean isFinishQueryAfterAnalyze(); + + default void setRealStatement(Statement realStatement) {} + + void setDataPartitionInfo(DataPartition dataPartition); + SchemaPartition getSchemaPartitionInfo(); + void setSchemaPartitionInfo(SchemaPartition schemaPartition); + DataPartition getDataPartitionInfo(); void setRedirectNodeList(List<TEndPoint> redirectNodeList); diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java index 6da6b261030,64f5fcefdc0..c9014eb6500 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java @@@ -202,8 -217,7 +217,7 @@@ public class LoadTsfileAnalyzer impleme LOGGER.info("Load - Analysis Stage: all tsfiles have been analyzed."); // data partition will be queried in the scheduler - final Analysis analysis = new Analysis(); - analysis.setStatement(loadTsFileStatement); + analysis.setRealStatement(loadTsFileStatement); return analysis; } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java index 0700aa5da51,1f69fd94917..d26348c5ffb --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java @@@ -23,10 -23,8 +23,9 @@@ import org.apache.iotdb.commons.schema. import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.tsfile.file.metadata.IDeviceID; - import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATER_NO_REGEX; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 4591ff7afc7,ca6e09c55b3..e504de0cc9e --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@@ -112,7 -113,7 +113,8 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; + import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java index 91acf28da3f,b369347bdff..0f300b780d0 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java @@@ -19,11 -19,9 +19,12 @@@ package org.apache.iotdb.db.queryengine.plan.relational.metadata; +import org.apache.iotdb.commons.partition.DataPartition; +import org.apache.iotdb.commons.partition.DataPartitionQueryParam; + import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputationWithAutoCreation; import org.apache.iotdb.db.queryengine.plan.relational.function.OperatorType; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@@ -101,30 -100,33 +103,45 @@@ public interface Metadata */ void validateDeviceSchema(ITableDeviceSchemaValidation schemaValidation, MPPQueryContext context); - /** - * Fetch and compute the schema of target timeseries, with device and measurement defined in given - * schemaComputationWithAutoCreation. The computation defined in given - * schemaComputationWithAutoCreation will be executed during scanning the fetched schema. If some - * target timeseries doesn't exist, they will be auto created. - * - * @param schemaComputationWithAutoCreationList define the target devices, measurements and - * computation - */ - default void fetchAndComputeSchemaWithAutoCreate( - List<? extends ISchemaComputationWithAutoCreation> schemaComputationWithAutoCreationList, - MPPQueryContext context) { - throw new UnsupportedOperationException(); - } - + /** + * Get or create data partition, used in cluster write scenarios. if enableAutoCreateSchema is + * true and database/series/time slots not exists, then automatically create. + * + * @param dataPartitionQueryParams the list of DataPartitionQueryParams + * @param userName + */ + default DataPartition getOrCreateDataPartition( + List<DataPartitionQueryParam> dataPartitionQueryParams, String userName) { + throw new UnsupportedOperationException(); + } ++ + // ======================== Table Model Schema Partition Interface ======================== + /** + * Get or create schema partition, used in data insertion with enable_auto_create_schema is true. + * if schemaPartition does not exist, then automatically create. + * + * <p>The database shall start with "root.". Concat this to a user-provided db name if necessary. + * + * <p>The device id shall be [table, seg1, ....] + */ + SchemaPartition getOrCreateSchemaPartition( + String database, List<IDeviceID> deviceIDList, String userName); + + /** + * For data query with completed id. + * + * <p>The database is a user-provided db name. + * + * <p>The device id shall be [table, seg1, ....] + */ + SchemaPartition getSchemaPartition(String database, List<IDeviceID> deviceIDList); + + /** + * For data query with partial device id conditions. + * + * <p>The database is a user-provided db name. + * + * <p>The device id shall be [table, seg1, ....] + */ + SchemaPartition getSchemaPartition(String database); }
