This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch agg_table_scan in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a37c2f0adaf132a3a70e9e011633c76858fda7c6 Merge: d27be355be4 6742d4bdf82 Author: Beyyes <[email protected]> AuthorDate: Fri Sep 27 16:26:32 2024 +0800 merge master .github/workflows/dependency-check.yml | 59 +++ dependencies.json | 167 +++++++ .../iotdb/it/env/cluster/ClusterConstant.java | 4 +- .../org/apache/iotdb/it/env/cluster/EnvUtils.java | 18 +- .../apache/iotdb/db/it/IoTDBDatetimeFormatIT.java | 9 + .../apache/iotdb/db/it/path/IoTDBQuotedPathIT.java | 11 +- .../iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java | 20 +- .../db/it/IoTDBMultiIDsWithAttributesTableIT.java | 185 ++++++++ .../pool/IoTDBInsertTableSessionPoolIT.java | 245 +++++++++++ .../iotdb/session/it/IoTDBSessionRelationalIT.java | 76 ++++ .../iotdb/tool/tsfile/ImportTsFileScanTool.java | 4 +- .../org/apache/iotdb/isession/IPooledSession.java | 1 + .../java/org/apache/iotdb/isession/ISession.java | 9 + .../org/apache/iotdb/rpc/RedirectException.java | 15 + .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 17 + .../java/org/apache/iotdb/session/Session.java | 216 ++++++++- .../apache/iotdb/session/SessionConnection.java | 4 +- .../org/apache/iotdb/session/pool/SessionPool.java | 22 +- .../apache/iotdb/session/pool/SessionWrapper.java | 8 +- .../iotdb/session/SessionCacheLeaderTest.java | 271 +++++++++++- .../iotdb/session/SessionConnectionTest.java | 2 +- iotdb-core/ainode/pyproject.toml | 2 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 12 +- .../confignode/conf/ConfigNodeStartupCheck.java | 5 +- .../consensus/request/ConfigPhysicalPlan.java | 202 +++------ .../consensus/request/ConfigPhysicalPlanType.java | 1 + .../request/ConfigPhysicalPlanVisitor.java | 2 +- ...PipePlanV2.java => ConfigPhysicalReadPlan.java} | 14 +- .../read/ainode/GetAINodeConfigurationPlan.java | 12 +- .../request/read/auth/AuthorReadPlan.java | 143 ++++++ .../consensus/request/read/cq/ShowCQPlan.java} | 13 +- .../request/read/database/CountDatabasePlan.java | 51 +-- .../request/read/database/GetDatabasePlan.java | 6 +- .../datanode/GetDataNodeConfigurationPlan.java | 30 +- .../read/function/GetFunctionTablePlan.java | 18 +- .../request/read/function/GetUDFJarPlan.java | 42 +- .../request/read/model/GetModelInfoPlan.java | 10 +- .../request/read/model/ShowModelPlan.java | 10 +- .../read/partition/CountTimeSlotListPlan.java | 49 +-- .../read/partition/GetDataPartitionPlan.java | 66 +-- .../read/partition/GetNodePathsPartitionPlan.java | 33 +- .../partition/GetOrCreateDataPartitionPlan.java | 10 +- .../partition/GetOrCreateSchemaPartitionPlan.java | 3 +- .../read/partition/GetSchemaPartitionPlan.java | 58 +-- .../read/partition/GetSeriesSlotListPlan.java | 36 +- .../read/partition/GetTimeSlotListPlan.java | 49 +-- .../read/pipe/plugin/GetPipePluginJarPlan.java | 35 +- .../read/pipe/plugin/GetPipePluginTablePlan.java | 18 +- .../request/read/pipe/task/ShowPipePlanV2.java | 18 +- .../request/read/region/GetRegionIdPlan.java | 49 +-- .../request/read/region/GetRegionInfoListPlan.java | 47 +- .../read/subscription/ShowSubscriptionPlan.java | 18 +- .../request/read/subscription/ShowTopicPlan.java | 18 +- .../FetchTablePlan.java} | 20 +- .../request/read/table/ShowTablePlan.java | 27 +- .../read/template/CheckTemplateSettablePlan.java | 33 +- .../read/template/GetAllSchemaTemplatePlan.java | 16 +- .../read/template/GetAllTemplateSetInfoPlan.java | 16 +- .../read/template/GetPathsSetTemplatePlan.java | 33 +- .../read/template/GetSchemaTemplatePlan.java | 32 +- .../read/template/GetTemplateSetInfoPlan.java | 37 +- .../read/trigger/GetTransferringTriggersPlan.java | 18 +- .../request/read/trigger/GetTriggerJarPlan.java | 42 +- .../read/trigger/GetTriggerLocationPlan.java | 38 +- .../request/read/trigger/GetTriggerTablePlan.java | 38 +- .../consensus/request/read/ttl/ShowTTLPlan.java | 19 +- .../request/{ => write}/auth/AuthorPlan.java | 99 +---- .../consensus/request/write/cq/ShowCQPlan.java | 45 -- .../write/partition/RemoveRegionLocationPlan.java | 5 - .../request/write/sync/ShowPipePlanV1.java | 59 --- .../consensus/response/table/FetchTableResp.java | 44 ++ .../statemachine/ConfigRegionStateMachine.java | 22 +- .../iotdb/confignode/manager/ConfigManager.java | 36 +- .../apache/iotdb/confignode/manager/IManager.java | 11 +- .../iotdb/confignode/manager/ModelManager.java | 10 +- .../confignode/manager/PermissionManager.java | 9 +- .../iotdb/confignode/manager/ProcedureManager.java | 18 +- .../manager/consensus/ConsensusManager.java | 3 +- .../iotdb/confignode/manager/cq/CQManager.java | 2 +- .../manager/load/balancer/RouteBalancer.java | 3 - .../manager/partition/PartitionManager.java | 16 +- .../agent/runtime/PipeConfigNodeRuntimeAgent.java | 13 +- .../agent/runtime/PipeConfigRegionListener.java | 7 +- .../protocol/IoTDBConfigRegionConnector.java | 3 +- .../PipeConfigPhysicalPlanPatternParseVisitor.java | 2 +- .../receiver/protocol/IoTDBConfigNodeReceiver.java | 2 +- .../PipeConfigPhysicalPlanTSStatusVisitor.java | 2 +- .../manager/schema/ClusterSchemaManager.java | 16 + .../iotdb/confignode/persistence/AuthorInfo.java | 53 +-- .../persistence/executor/ConfigPlanExecutor.java | 17 +- .../confignode/persistence/node/NodeInfo.java | 33 +- .../confignode/persistence/pipe/PipeInfo.java | 33 +- .../schema/CNPhysicalPlanGenerator.java | 2 +- .../persistence/schema/ClusterSchemaInfo.java | 23 + .../confignode/persistence/schema/ConfigMTree.java | 17 + .../impl/schema/table/AddTableColumnProcedure.java | 1 - .../impl/schema/table/CreateTableProcedure.java | 1 - .../schema/table/SetTablePropertiesProcedure.java | 1 - .../impl/sync/AuthOperationProcedure.java | 2 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 23 +- .../request/ConfigPhysicalPlanSerDeTest.java | 482 +-------------------- ...eConfigPhysicalPlanPatternParseVisitorTest.java | 2 +- .../confignode/persistence/AuthorInfoTest.java | 82 ++-- .../persistence/CNPhysicalPlanGeneratorTest.java | 2 +- .../ConfigRegionListeningQueueTest.java | 2 +- .../pipe/receiver/PipeEnrichedProcedureTest.java | 2 +- .../impl/sync/AuthOperationProcedureTest.java | 29 +- .../apache/iotdb/consensus/ConsensusFactory.java | 23 +- .../consensus/config/PipeConsensusConfig.java | 26 +- .../apache/iotdb/consensus/iot/IoTConsensus.java | 4 + .../consensus/iot/logdispatcher/LogDispatcher.java | 11 +- .../iotdb/consensus/ratis/RatisConsensus.java | 6 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 73 ++-- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 57 ++- .../db/consensus/DataRegionConsensusImpl.java | 4 +- .../schemaregion/SchemaExecutionVisitor.java | 12 + .../agent/receiver/PipeDataNodeReceiverAgent.java | 2 +- .../client/IoTDBDataNodeSyncClientManager.java | 26 +- .../pipeconsensus/PipeConsensusAsyncConnector.java | 4 +- .../thrift/sync/IoTDBDataNodeSyncConnector.java | 3 +- .../twostage/plugin/TwoStageCountProcessor.java | 2 +- .../pipeconsensus/PipeConsensusReceiver.java | 13 +- .../transform/converter/ValueConverter.java | 12 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 9 + .../handler/AINodeRPCServiceThriftHandler.java | 60 --- .../protocol/thrift/impl/AINodeRPCServiceImpl.java | 187 -------- .../db/queryengine/common/MPPQueryContext.java | 18 +- .../execution/executor/RegionReadExecutor.java | 3 +- .../fragment/FragmentInstanceContext.java | 5 +- .../process/AggregationMergeSortOperator.java | 3 +- ...erator.java => TableFullOuterJoinOperator.java} | 268 ++++++++---- ...inOperator.java => TableInnerJoinOperator.java} | 116 ++--- .../relational/ColumnTransformerBuilder.java | 23 +- .../iotdb/db/queryengine/plan/Coordinator.java | 6 +- .../db/queryengine/plan/analyze/Analysis.java | 1 + .../db/queryengine/plan/analyze/IAnalysis.java | 2 + .../analyze/lock/DataNodeSchemaLockManager.java | 17 +- .../execution/config/TableConfigTaskVisitor.java | 90 +++- .../execution/config/TreeConfigTaskVisitor.java | 16 +- .../config/executor/ClusterConfigTaskExecutor.java | 135 ++---- .../config/executor/IConfigTaskExecutor.java | 24 +- .../config/metadata/CreatePipePluginTask.java | 10 + .../config/metadata/DropPipePluginTask.java | 7 + .../metadata/{model => }/ShowAINodesTask.java | 11 +- .../config/metadata/ShowDataNodesTask.java | 2 +- .../execution/config/metadata/ShowRegionTask.java | 29 +- ...ShowDataNodesTask.java => ShowAINodesTask.java} | 11 +- .../metadata/relational/ShowConfigNodesTask.java | 9 +- .../metadata/relational/ShowDataNodesTask.java | 9 +- .../execution/config/sys/pipe/AlterPipeTask.java | 14 + .../execution/config/sys/pipe/CreatePipeTask.java | 11 + .../execution/config/sys/pipe/DropPipeTask.java | 8 + .../execution/config/sys/pipe/ShowPipeTask.java | 7 + .../execution/config/sys/pipe/StartPipeTask.java | 7 + .../execution/config/sys/pipe/StopPipeTask.java | 7 + .../db/queryengine/plan/parser/ASTVisitor.java | 6 +- .../plan/parser/StatementGenerator.java | 11 +- .../plan/planner/OperatorTreeGenerator.java | 2 +- .../plan/planner/TableOperatorGenerator.java | 22 +- .../plan/planner/plan/node/write/InsertNode.java | 1 - .../planner/plan/node/write/InsertTabletNode.java | 10 +- .../node/write/RelationalInsertTabletNode.java | 43 ++ .../plan/relational/analyzer/Analysis.java | 5 + .../relational/analyzer/ExpressionTreeUtils.java | 5 +- .../relational/analyzer/StatementAnalyzer.java | 59 ++- .../plan/relational/metadata/Metadata.java | 3 +- .../metadata/TableBuiltinAggregationFunction.java | 30 +- .../relational/metadata/TableMetadataImpl.java | 30 +- .../fetcher/TableHeaderSchemaValidator.java | 60 ++- .../plan/relational/planner/RelationPlanner.java | 17 +- .../plan/relational/planner/TableModelPlanner.java | 40 +- .../distribute/TableDistributedPlanner.java | 7 + .../TableModelTypeProviderExtractor.java | 11 +- .../PushAggregationIntoTableScan.java | 14 +- .../PushLimitOffsetIntoTableScan.java | 6 +- .../optimizations/PushPredicateIntoTableScan.java | 9 +- .../relational/planner/optimizations/Util.java | 7 +- .../plan/relational/sql/ast/AlterPipe.java | 143 ++++++ .../plan/relational/sql/ast/AstVisitor.java | 40 ++ .../relational/sql/ast/CoalesceExpression.java | 28 ++ .../plan/relational/sql/ast/CreatePipe.java | 113 +++++ .../plan/relational/sql/ast/CreatePipePlugin.java | 95 ++++ .../plan/relational/sql/ast/DropPipe.java | 75 ++++ .../plan/relational/sql/ast/DropPipePlugin.java | 75 ++++ .../plan/relational/sql/ast/Expression.java | 3 + .../plan/relational/sql/ast/PipeStatement.java} | 18 +- .../plan/relational/sql/ast/ShowAINodes.java} | 41 +- .../plan/relational/sql/ast/ShowPipePlugins.java} | 20 +- .../plan/relational/sql/ast/ShowPipes.java | 76 ++++ .../plan/relational/sql/ast/StartPipe.java | 65 +++ .../plan/relational/sql/ast/StopPipe.java | 65 +++ .../relational/sql/ast/TableExpressionType.java | 3 +- .../plan/relational/sql/parser/AstBuilder.java | 218 ++++++++++ .../plan/relational/sql/util/SqlFormatter.java | 208 +++++++++ .../plan/relational/type/InternalTypeManager.java | 1 + .../column/multi/CoalesceColumnTransformer.java | 58 +++ .../schemaregion/utils/MetaFormatUtils.java | 5 +- .../db/schemaengine/table/DataNodeTableCache.java | 217 ++++++++-- .../apache/iotdb/db/service/AINodeRPCService.java | 94 ---- .../java/org/apache/iotdb/db/service/DataNode.java | 7 +- .../iotdb/db/storageengine/StorageEngine.java | 4 +- .../db/storageengine/dataregion/DataRegion.java | 17 +- .../RepairUnsortedFileCompactionPerformer.java | 9 +- .../execute/task/AbstractCompactionTask.java | 4 +- .../execute/task/CrossSpaceCompactionTask.java | 3 + .../task/InsertionCrossSpaceCompactionTask.java | 2 + .../task/RepairUnsortedFileCompactionTask.java | 88 ++-- .../execute/task/SettleCompactionTask.java | 30 +- .../utils/AlignedSeriesBatchCompactionUtils.java | 10 +- .../compaction/repair/RepairDataFileScanUtil.java | 22 +- .../repair/RepairTimePartitionScanTask.java | 5 +- .../repair/UnsortedFileRepairTaskScheduler.java | 2 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 4 + .../impl/SizeTieredCompactionSelector.java | 12 +- .../dataregion/tsfile/TsFileRepairStatus.java | 16 +- .../dataregion/tsfile/TsFileResource.java | 2 +- .../FileTimeIndexCacheWriter.java | 8 +- .../storageengine/dataregion/wal/WALManager.java | 18 +- .../dataregion/wal/recover/WALNodeRecoverTask.java | 7 +- .../agent/SubscriptionBrokerAgent.java | 10 + .../db/subscription/broker/SubscriptionBroker.java | 20 + .../broker/SubscriptionPrefetchingQueue.java | 14 +- .../db/subscription/event/SubscriptionEvent.java | 6 + .../batch/SubscriptionPipeTabletEventBatch.java | 10 +- .../batch/SubscriptionPipeTsFileEventBatch.java | 12 +- .../event/pipe/SubscriptionPipeEmptyEvent.java | 9 + .../event/pipe/SubscriptionPipeEvents.java | 4 + .../pipe/SubscriptionPipeTabletBatchEvents.java | 9 + .../pipe/SubscriptionPipeTsFileBatchEvents.java | 13 + .../pipe/SubscriptionPipeTsFilePlainEvent.java | 9 + .../SubscriptionPrefetchingQueueMetrics.java | 2 +- .../task/subtask/SubscriptionConnectorSubtask.java | 14 + .../SubscriptionConnectorSubtaskManager.java | 3 + .../iotdb/db/utils/constant/SqlConstant.java | 5 + .../operator/AggregationMergeSortOperatorTest.java | 177 ++++++++ .../plan/relational/analyzer/AggregationTest.java | 171 ++++++-- .../plan/relational/analyzer/JoinTest.java | 10 +- .../plan/relational/analyzer/TSBSMetadata.java | 5 +- .../plan/relational/analyzer/TSBSTest.java | 1 + .../plan/relational/analyzer/TestMatadata.java | 5 +- .../assertions/AggregationTableScanMatcher.java | 11 +- .../planner/assertions/PlanMatchPattern.java | 2 +- .../repair/RepairUnsortedFileCompactionTest.java | 100 ++++- .../repair/RepairUnsortedFileSchedulerTest.java | 2 +- .../tablemodel/CompactionWithAllNullRowsTest.java | 2 + .../compaction/utils/BatchCompactionUtilsTest.java | 138 ++++++ .../compaction/utils/CompactionTaskQueueTest.java | 8 +- .../conf/iotdb-system.properties.template | 33 +- .../assembly/resources/sbin/destroy-datanode.bat | 12 +- .../assembly/resources/sbin/destroy-datanode.sh | 4 +- .../agent/plugin/builtin/BuiltinPipePlugin.java | 6 +- .../config/constant/PipeExtractorConstant.java | 6 + .../connector/protocol/IoTDBSslSyncConnector.java | 29 +- .../options/PipeInclusionOptions.java | 2 +- .../apache/iotdb/commons/schema/table/TsTable.java | 4 +- .../schema/table/TsTableInternalRPCUtil.java | 94 +++- .../org/apache/iotdb/commons/utils/PathUtils.java | 7 + .../db/relational/grammar/sql/RelationalSql.g4 | 144 +++++- .../src/main/thrift/confignode.thrift | 7 + .../src/main/thrift/datanode.thrift | 15 +- pom.xml | 89 +++- src/main/groovy/checkDependencies.groovy | 60 +++ src/main/xslt/sbom-filter.xsl | 41 ++ 263 files changed, 5921 insertions(+), 3182 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 6d9667235d7,b0b520c92bb..288802a4537 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@@ -49,8 -49,8 +49,9 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.execution.operator.sink.IdentitySinkOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; - import org.apache.iotdb.db.queryengine.execution.operator.source.relational.InnerJoinOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableAggregationTableScanOperator; + import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableFullOuterJoinOperator; + import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableInnerJoinOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator; import org.apache.iotdb.db.queryengine.execution.relational.ColumnTransformerBuilder; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 9e8e5dfe70d,43040121845..5d86d132573 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@@ -543,20 -543,26 +543,17 @@@ public class TableMetadataImpl implemen functionName)); } break; - case SqlConstant.MIN_TIME: - case SqlConstant.MAX_TIME: - case SqlConstant.FIRST_VALUE: - case SqlConstant.LAST_VALUE: - case SqlConstant.TIME_DURATION: case SqlConstant.MODE: + case SqlConstant.FIRST: + case SqlConstant.LAST: if (argumentTypes.size() != 1) { throw new SemanticException( String.format( "Aggregate functions [%s] should only have one argument", functionName)); } break; - case SqlConstant.FIRST: - case SqlConstant.LAST: - if (argumentTypes.size() != 2) { - throw new SemanticException( - String.format( - "Aggregate functions [%s] should only have two arguments", functionName)); - } else if (!isTimestampType(argumentTypes.get(1))) { - throw new SemanticException( - String.format( - "Second argument of Aggregate functions [%s] should be orderable", functionName)); - } + case SqlConstant.FIRST_BY: + case SqlConstant.LAST_BY: case SqlConstant.MAX_BY: case SqlConstant.MIN_BY: if (argumentTypes.size() != 2) {
