This is an automated email from the ASF dual-hosted git repository. lancelly pushed a commit to branch support_uncorrelated_quantified_comparison in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 981fdb6a58acba20c0c5ff3695166cb281eac1c4 Merge: 44dab8ecc27 e92be6f63ab Author: lancelly <[email protected]> AuthorDate: Fri Jan 10 11:01:28 2025 +0800 fix conflicts .github/workflows/pipe-it-2cluster.yml | 18 +- example/udf/pom.xml | 6 +- .../apache/iotdb/udf/AggregateFunctionExample.java | 35 +- .../apache/iotdb/udf/ScalarFunctionExample.java | 57 +-- .../db/query/udf/example/relational/AllSum.java | 49 +- .../query/udf/example/relational/ContainNull.java | 21 +- .../relational/{DatePlusOne.java => DatePlus.java} | 31 +- .../query/udf/example/relational/FirstTwoSum.java | 33 +- .../db/query/udf/example/relational/MyAvg.java | 32 +- .../db/query/udf/example/relational/MyCount.java | 19 +- .../iotdb/it/utils/TsFileTableGenerator.java | 202 ++++++++ .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 80 ++- .../iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java | 3 +- .../it/autocreate/IoTDBPipeSwitchStatusIT.java | 23 +- .../pipe/it/manual/IoTDBPipeTypeConversionIT.java | 2 +- .../iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java | 30 +- .../pipe/it/tablemodel/IoTDBPipeClusterIT.java | 290 ++++++----- .../pipe/it/tablemodel/IoTDBPipeDataSinkIT.java | 429 +++++++++++++++- .../it/tablemodel/IoTDBPipeDoubleLivingIT.java | 329 ++++++++++++ .../pipe/it/tablemodel/IoTDBPipeIsolationIT.java | 377 ++++++++++++++ .../pipe/it/tablemodel/IoTDBPipeNullValueIT.java | 4 +- .../it/tablemodel/IoTDBPipeSwitchStatusIT.java | 23 +- .../pipe/it/tablemodel/IoTDBPipeSyntaxIT.java | 33 +- .../pipe/it/tablemodel/IoTDBPipeWithLoadIT.java | 8 +- .../iotdb/pipe/it/tablemodel/TableModelUtils.java | 563 +++++++++++++++++---- .../it/db/it/udf/IoTDBSQLFunctionManagementIT.java | 19 +- .../it/udf/IoTDBUserDefinedScalarFunctionIT.java | 2 +- .../it/query/recent/IoTDBTableAggregationIT.java | 339 +++++++++++++ .../IoTDBUncorrelatedInPredicateSubqueryIT.java | 2 +- .../relational/it/schema/IoTDBDatabaseIT.java | 41 ++ .../it/session/IoTDBSessionRelationalIT.java | 156 ++++-- .../analysis/AggregateFunctionAnalysis.java | 62 +++ .../api/customizer/analysis/FunctionAnalysis.java | 15 +- .../analysis/ScalarFunctionAnalysis.java | 51 ++ ...ctionParameters.java => FunctionArguments.java} | 28 +- .../exception/UDFArgumentNotValidException.java | 10 +- .../udf/api/relational/AggregateFunction.java | 48 +- .../iotdb/udf/api/relational/ScalarFunction.java | 47 +- .../apache/iotdb/tool/data/AbstractDataTool.java | 32 +- .../org/apache/iotdb/tool/data/ExportData.java | 43 +- .../org/apache/iotdb/tool/data/ImportData.java | 24 +- .../client-py/table_model_session_example.py | 12 +- .../client-py/table_model_session_pool_example.py | 8 +- .../payload/SubscriptionFileHandler.java | 24 +- .../org/apache/iotdb/session/util/RetryUtils.java | 30 +- .../consensus/request/ConfigPhysicalPlanType.java | 2 + .../table/DescTable4InformationSchemaPlan.java} | 12 +- .../table/ShowTable4InformationSchemaPlan.java} | 11 +- .../response/pipe/task/PipeTableResp.java | 7 + .../table/DescTable4InformationSchemaResp.java | 42 ++ .../table/ShowTable4InformationSchemaResp.java | 44 ++ .../iotdb/confignode/manager/ConfigManager.java | 32 +- .../apache/iotdb/confignode/manager/IManager.java | 31 +- .../iotdb/confignode/manager/load/LoadManager.java | 11 + .../confignode/manager/load/cache/LoadCache.java | 17 + .../iotdb/confignode/manager/node/NodeManager.java | 13 + .../pipe/coordinator/task/PipeTaskCoordinator.java | 74 ++- .../manager/schema/ClusterSchemaManager.java | 32 ++ .../persistence/executor/ConfigPlanExecutor.java | 4 + .../confignode/persistence/pipe/PipeTaskInfo.java | 21 +- .../persistence/schema/ClusterSchemaInfo.java | 77 +++ .../confignode/persistence/schema/ConfigMTree.java | 16 + .../impl/pipe/task/AlterPipeProcedureV2.java | 4 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 32 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +- .../schemaregion/SchemaExecutionVisitor.java | 15 +- .../LoadAnalyzeException.java} | 8 +- .../LoadAnalyzeTableColumnDisorderException.java} | 7 +- .../LoadAnalyzeTypeMismatchException.java} | 6 +- .../batch/PipeTabletEventTsFileBatch.java | 384 +++----------- .../request/PipeTransferTabletRawReq.java | 4 +- .../request/PipeTransferTabletRawReqV2.java | 9 +- .../connector/protocol/opcua/OpcUaNameSpace.java | 8 +- .../request/PipeConsensusDeleteNodeReq.java | 4 +- .../async/IoTDBDataRegionAsyncConnector.java | 10 +- .../PipeTransferTabletBatchEventHandler.java | 2 +- .../async/handler/PipeTransferTsFileHandler.java | 13 +- .../thrift/sync/IoTDBDataRegionSyncConnector.java | 19 +- .../util/builder/PipeTableModeTsFileBuilder.java | 273 ++++++++++ .../util/builder/PipeTreeModelTsFileBuilder.java | 268 ++++++++++ .../connector/util/builder/PipeTsFileBuilder.java | 162 ++++++ .../util/{ => cacher}/LeaderCacheUtils.java | 2 +- .../sorter/PipeTableModelTabletEventSorter.java | 271 ++++++++++ .../util/{ => sorter}/PipeTabletEventSorter.java | 107 +--- .../sorter/PipeTreeModelTabletEventSorter.java | 121 +++++ .../dataregion/IoTDBDataRegionExtractor.java | 20 +- .../PipeDataNodeRemainingEventAndTimeOperator.java | 20 +- .../pipeconsensus/PipeConsensusReceiver.java | 31 +- .../protocol/thrift/IoTDBDataNodeReceiver.java | 7 +- .../PipeConvertedInsertTabletStatement.java | 4 + ...leStatementDataTypeConvertExecutionVisitor.java | 7 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 32 +- .../InformationSchemaContentSupplierFactory.java | 372 ++++++++++++-- .../relational/aggregation/AccumulatorFactory.java | 23 +- .../relational/aggregation/CountIfAccumulator.java | 101 ++++ .../UserDefinedAggregateFunctionAccumulator.java | 27 +- .../grouped/GroupedCountIfAccumulator.java | 79 +++ .../relational/ColumnTransformerBuilder.java | 14 +- .../iotdb/db/queryengine/plan/Coordinator.java | 4 + .../db/queryengine/plan/analyze/Analysis.java | 10 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 57 ++- .../plan/analyze/load/LoadTsFileAnalyzer.java | 44 +- .../analyze/load/LoadTsFileTableSchemaCache.java | 14 +- .../load/LoadTsFileToTableModelAnalyzer.java | 27 +- .../load/LoadTsFileToTreeModelAnalyzer.java | 17 +- .../load/TreeSchemaAutoCreatorAndVerifier.java | 30 +- .../config/executor/ClusterConfigTaskExecutor.java | 20 +- .../config/metadata/relational/ShowDBTask.java | 4 +- .../execution/config/sys/pipe/AlterPipeTask.java | 8 +- .../execution/config/sys/pipe/DropPipeTask.java | 7 +- .../execution/config/sys/pipe/ShowPipeTask.java | 3 +- .../execution/config/sys/pipe/StartPipeTask.java | 7 +- .../execution/config/sys/pipe/StopPipeTask.java | 7 +- .../db/queryengine/plan/parser/ASTVisitor.java | 9 +- .../plan/planner/LogicalPlanBuilder.java | 2 +- .../plan/planner/TableOperatorGenerator.java | 35 +- .../plan/planner/plan/node/PlanVisitor.java | 2 +- .../relational/analyzer/StatementAnalyzer.java | 12 +- .../plan/relational/metadata/Metadata.java | 4 +- .../relational/metadata/TableMetadataImpl.java | 42 +- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 10 +- .../fetcher/TableHeaderSchemaValidator.java | 8 +- .../plan/relational/planner/QueryPlanner.java | 2 +- .../plan/relational/planner/node/SemiJoinNode.java | 28 + .../DataNodeLocationSupplierFactory.java | 51 +- .../sql/ast/AbstractQueryDeviceWithCache.java | 2 + .../plan/relational/sql/ast/AlterPipe.java | 22 +- .../plan/relational/sql/ast/AstVisitor.java | 4 + .../sql/ast/{DropPipe.java => CountStatement.java} | 55 +- .../plan/relational/sql/ast/DropPipe.java | 10 +- .../plan/relational/sql/ast/LoadTsFile.java | 14 +- .../plan/relational/sql/ast/ShowPipes.java | 10 +- .../plan/relational/sql/ast/StartPipe.java | 8 +- .../plan/relational/sql/ast/StopPipe.java | 8 +- .../plan/relational/sql/parser/AstBuilder.java | 36 +- .../plan/relational/sql/rewrite/ShowRewrite.java | 51 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 80 ++- .../plan/statement/crud/LoadTsFileStatement.java | 9 + .../metadata/pipe/AlterPipeStatement.java | 31 +- .../statement/metadata/pipe/DropPipeStatement.java | 23 +- .../metadata/pipe/ShowPipesStatement.java | 18 +- .../metadata/pipe/StartPipeStatement.java | 17 +- .../statement/metadata/pipe/StopPipeStatement.java | 17 +- .../schemaengine/table/InformationSchemaUtils.java | 9 +- .../db/storageengine/dataregion/DataRegion.java | 135 ++--- .../CompactionSourceFileDeletedException.java} | 13 +- .../execute/task/InnerSpaceCompactionTask.java | 6 +- .../schedule/CompactionScheduleContext.java | 21 + .../compaction/schedule/CompactionScheduler.java | 60 ++- .../estimator/AbstractCrossSpaceEstimator.java | 7 +- .../estimator/AbstractInnerSpaceEstimator.java | 8 +- .../estimator/CompactionEstimateUtils.java | 35 +- .../FastCompactionInnerCompactionEstimator.java | 16 +- .../FastCrossSpaceCompactionEstimator.java | 17 +- .../selector/estimator/MetadataInfo.java | 42 ++ .../ReadChunkInnerCompactionEstimator.java | 15 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 16 +- .../db/storageengine/load/LoadTsFileManager.java | 21 +- .../load/active/ActiveLoadDirScanner.java | 27 +- .../load/active/ActiveLoadTsFileLoader.java | 13 +- .../LoadConvertedInsertTabletStatement.java | 18 +- ...leStatementDataTypeConvertExecutionVisitor.java | 10 +- ...eeStatementDataTypeConvertExecutionVisitor.java | 3 +- .../converter/LoadTsFileDataTypeConverter.java | 103 ++-- .../load/metrics/LoadTsFileCostMetricsSet.java | 13 +- .../batch/SubscriptionPipeTsFileEventBatch.java | 11 +- .../org/apache/iotdb/db/utils/CommonUtils.java | 6 - .../pipe/connector/PipeTabletEventSorterTest.java | 210 +++++++- .../cross/CrossSpaceCompactionSelectorTest.java | 102 +--- .../InsertionCrossSpaceCompactionSelectorTest.java | 4 +- .../cross/InsertionCrossSpaceCompactionTest.java | 64 ++- .../utils/CompactionTaskMemCostEstimatorTest.java | 48 ++ .../conf/iotdb-system.properties.template | 6 +- .../apache/iotdb/commons/cluster/NodeStatus.java | 13 + .../commons/pipe/agent/task/meta/PipeMeta.java | 40 ++ .../pipe/agent/task/meta/PipeMetaKeeper.java | 8 + .../config/constant/PipeExtractorConstant.java | 3 + .../pipe/datastructure/pattern/TablePattern.java | 21 +- .../pipe/datastructure/pattern/TreePattern.java | 22 +- .../commons/pipe/extractor/IoTDBExtractor.java | 65 ++- .../commons/pipe/receiver/IoTDBFileReceiver.java | 17 +- .../commons/pipe/receiver/IoTDBReceiverAgent.java | 7 +- .../snapshot/PipeSnapshotResourceManager.java | 9 +- .../TableBuiltinAggregationFunction.java | 2 + .../org/apache/iotdb/commons/utils/FileUtils.java | 49 ++ .../apache/iotdb/commons/utils/RetryUtils.java} | 30 +- .../src/main/thrift/confignode.thrift | 42 +- 187 files changed, 7047 insertions(+), 1825 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java index ec5d383df7e,fae80f7c69f..09d9fb23b74 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java @@@ -208,8 -208,8 +209,10 @@@ public class AccumulatorFactory switch (aggregationType) { case COUNT: return new CountAccumulator(); + case COUNT_ALL: + return new CountAllAccumulator(); + case COUNT_IF: + return new CountIfAccumulator(); case AVG: return new AvgAccumulator(inputDataTypes.get(0)); case SUM: diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 52d60bc29c3,df229a1598c..9ca7ddda112 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@@ -1525,11 -1532,9 +1532,11 @@@ public class TableOperatorGenerator ext Type sourceJoinKeyType = context.getTypeProvider().getTableModelType(node.getSourceJoinSymbol()); + - checkArgument( - sourceJoinKeyType - == context.getTypeProvider().getTableModelType(node.getFilteringSourceJoinSymbol()), - "Join key type mismatch."); + checkIfJoinKeyTypeMatches( + sourceJoinKeyType, + context.getTypeProvider().getTableModelType(node.getFilteringSourceJoinSymbol())); ++ OperatorContext operatorContext = context .getDriverContext() diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index ba53e684be5,ab2ba5b9f33..88edf9c7d73 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@@ -615,7 -624,7 +624,8 @@@ public class TableMetadataImpl implemen // get return type switch (functionName.toLowerCase(Locale.ENGLISH)) { case SqlConstant.COUNT: + case SqlConstant.COUNT_ALL: + case SqlConstant.COUNT_IF: return INT64; case SqlConstant.FIRST_AGGREGATION: case SqlConstant.LAST_AGGREGATION: diff --cc iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinAggregationFunction.java index 980e2b1bfeb,10aa13ed4ad..3d0510957d1 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinAggregationFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinAggregationFunction.java @@@ -38,7 -38,7 +38,8 @@@ import static org.apache.tsfile.read.co public enum TableBuiltinAggregationFunction { SUM("sum"), COUNT("count"), + COUNT_ALL("count_all"), + COUNT_IF("count_if"), AVG("avg"), EXTREME("extreme"), MAX("max"), @@@ -82,7 -82,7 +83,8 @@@ final String functionName = name.toLowerCase(); switch (functionName) { case "count": + case "count_all": + case "count_if": return INT64; case "sum": return DOUBLE;
