This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/TableModelGrammar in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cd254c2fdd3920b05b06bc6d3b143a6f361cefc4 Merge: 4676d100dbf c3c48a2e321 Author: JackieTien97 <[email protected]> AuthorDate: Thu Feb 29 10:26:01 2024 +0800 Merge remote-tracking branch 'origin/master' into ty/TableModelGrammar .github/workflows/client-cpp.yml | 10 +- .github/workflows/client-go.yml | 6 +- .github/workflows/client-python.yml | 4 +- .github/workflows/cluster-it-1c1d.yml | 8 +- .github/workflows/cluster-it-1c3d.yml | 22 +- .github/workflows/grafana-plugin.yml | 10 +- .github/workflows/pipe-it-2cluster.yml | 6 +- .github/workflows/sonar-codecov.yml | 12 +- .github/workflows/unit-test.yml | 6 +- .../java/org/apache/iotdb/udf/UDAFExample.java | 198 +++++++++ integration-test/pom.xml | 14 +- .../apache/iotdb/db/query/udf/example/UDAFAvg.java | 190 ++++++++ .../iotdb/db/query/udf/example/UDAFCount.java | 112 +++++ .../apache/iotdb/db/query/udf/example/UDAFSum.java | 189 ++++++++ .../it/env/cluster/config/MppCommonConfig.java | 6 + .../env/cluster/config/MppSharedCommonConfig.java | 7 + .../iotdb/it/env/cluster/env/AbstractEnv.java | 188 +++++--- .../it/env/remote/config/RemoteCommonConfig.java | 5 + .../iotdb/it/env/remote/env/RemoteServerEnv.java | 20 + .../constant/BuiltinAggregationFunctionEnum.java | 3 +- .../apache/iotdb/itbase/constant/TestConstant.java | 12 + .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 24 +- .../org/apache/iotdb/itbase/env/CommonConfig.java | 2 + .../it/cluster/IoTDBClusterRestartIT.java | 42 +- .../confignode/it/cluster/IoTDBClusterStartIT.java | 28 +- .../confignode/it/procedure/IoTDBProcedureIT.java | 137 ++++++ .../db/it/IOTDBInsertWithTimeAtAnyIndexIT.java | 99 +++++ .../iotdb/db/it/IoTDBInsertWithoutTimeIT.java | 8 +- .../org/apache/iotdb/db/it/IoTDBRecoverIT.java | 307 +++++-------- .../apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java | 166 +++---- .../org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java | 20 - .../it/IoTDBSyntaxConventionStringLiteralIT.java | 8 +- .../db/it/aggregation/IoTDBAggregationIT.java | 52 +++ .../aggregation/IoTDBCountTimeAlignedDeviceIT.java | 2 +- .../db/it/aggregation/maxby/IoTDBMaxByIT.java | 3 +- .../db/it/aggregation/minby/IoTDBMinBy2IT.java | 25 +- .../db/it/aggregation/minby/IoTDBMinBy3IT.java | 32 +- .../minby/IoTDBMinByAlignedSeriesIT.java | 71 +++ .../IoTDBMaxByIT.java => minby/IoTDBMinByIT.java} | 131 +++--- .../db/it/aligned/IoTDBAlignedSeriesQueryIT.java | 48 ++ .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 2 + .../org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java | 10 +- .../apache/iotdb/db/it/cq/IoTDBCQExecInNsIT.java | 10 +- .../apache/iotdb/db/it/cq/IoTDBCQExecInUsIT.java | 10 +- .../db/it/udaf/IoTDBUDAFGroupByConditionIT.java | 325 ++++++++++++++ .../iotdb/db/it/udaf/IoTDBUDAFGroupByCountIT.java | 292 ++++++++++++ .../iotdb/db/it/udaf/IoTDBUDAFGroupByLevelIT.java | 495 +++++++++++++++++++++ .../db/it/udaf/IoTDBUDAFGroupBySessionIT.java | 302 +++++++++++++ .../iotdb/db/it/udaf/IoTDBUDAFGroupByTimeIT.java | 340 ++++++++++++++ .../db/it/udaf/IoTDBUDAFGroupByVariationIT.java | 494 ++++++++++++++++++++ .../apache/iotdb/db/it/udaf/IoTDBUDAFHavingIT.java | 287 ++++++++++++ .../iotdb/db/it/udaf/IoTDBUDAFManagementIT.java | 336 ++++++++++++++ .../apache/iotdb/db/it/udaf/IoTDBUDAFMiscIT.java | 126 ++++++ .../iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java | 440 ++++++++++++++++++ .../iotdb/db/it/udaf/IoTDBUDAFOrderByIT.java | 468 +++++++++++++++++++ .../org/apache/iotdb/db/it/utils/TestUtils.java | 21 +- .../apache/iotdb/pipe/it/IoTDBPipeClusterIT.java | 38 +- .../apache/iotdb/pipe/it/IoTDBPipeLifeCycleIT.java | 25 +- .../test/java/org/apache/iotdb/spark/it/Utils.java | 1 - .../java/org/apache/iotdb/tsfile/utils/BitMap.java | 0 .../main/java/org/apache/iotdb/udf/api/State.java | 28 +- .../main/java/org/apache/iotdb/udf/api/UDAF.java | 71 +++ .../api/customizer/config/UDAFConfigurations.java | 17 +- .../apache/iotdb/udf/api/utils/ResultValue.java | 76 ++++ .../cli/src/assembly/resources/sbin/start-cli.bat | 2 + .../src/assembly/resources/tools/collect-info.bat | 2 + .../src/assembly/resources/tools/export-csv.bat | 3 + .../src/assembly/resources/tools/export-tsfile.bat | 3 + .../src/assembly/resources/tools/import-csv.bat | 3 + .../src/assembly/resources/tools/load-tsfile.bat | 2 + .../java/org/apache/iotdb/tool/ExportTsFile.java | 3 +- .../org/apache/iotdb/jdbc/IoTDBConnection.java | 4 +- .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 2 +- .../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 4 +- .../sql/factory/IoTDBDynamicTableFactory.java | 1 + iotdb-connector/grafana-plugin/yarn.lock | 6 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 18 +- .../assembly/resources/sbin/start-confignode.bat | 4 + .../iotdb/confignode/manager/ClusterManager.java | 23 + .../iotdb/confignode/manager/ConfigManager.java | 7 + .../apache/iotdb/confignode/manager/IManager.java | 9 + .../iotdb/confignode/manager/ProcedureManager.java | 13 +- .../manager/load/balancer/PartitionBalancer.java | 31 +- .../iotdb/confignode/manager/node/NodeManager.java | 37 +- .../confignode/persistence/ProcedureInfo.java | 7 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 4 +- .../iotdb/confignode/procedure/Procedure.java | 15 + .../confignode/procedure/ProcedureExecutor.java | 2 + .../impl/CreateManyDatabasesProcedure.java | 116 +++++ .../procedure/store/ConfigProcedureStore.java | 9 + .../procedure/store/ProcedureFactory.java | 9 +- .../confignode/procedure/store/ProcedureStore.java | 192 -------- .../confignode/procedure/store/ProcedureType.java | 5 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 5 + ...TestSTMProcedure.java => STMProcedureTest.java} | 2 +- .../procedure/store/TestProcedureStore.java | 113 ----- .../resources/conf/iotdb-datanode.properties | 6 +- .../src/assembly/resources/sbin/start-datanode.bat | 5 + .../src/main/codegen/templates/ModeAccumulator.ftl | 5 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 34 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 30 +- .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 21 - .../db/pipe/agent/task/PipeTaskDataNodeAgent.java | 30 +- .../protocol/airgap/IoTDBAirGapConnector.java | 34 +- .../async/IoTDBThriftAsyncClientManager.java | 6 +- .../thrift/async/IoTDBThriftAsyncConnector.java | 46 +- .../PipeTransferTabletBatchEventHandler.java | 9 +- .../PipeTransferTabletInsertionEventHandler.java | 4 +- .../thrift/sync/IoTDBThriftSyncClientManager.java | 7 +- .../thrift/sync/IoTDBThriftSyncConnector.java | 34 -- .../protocol/writeback/WriteBackConnector.java | 20 - .../apache/iotdb/db/pipe/event/EnrichedEvent.java | 42 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 18 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 25 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 31 +- .../tsfile/TsFileInsertionDataContainer.java | 16 + .../PipeRealtimeDataRegionHybridExtractor.java | 22 +- .../PipeRealtimeDataRegionLogExtractor.java | 12 +- .../pipe/extractor/realtime/epoch/TsFileEpoch.java | 14 + .../realtime/epoch/TsFileEpochManager.java | 5 +- .../receiver/thrift/IoTDBThriftReceiverV1.java | 13 +- .../db/pipe/resource/PipeResourceManager.java | 7 + .../resource/log/PipeLogManager.java} | 26 +- .../iotdb/db/pipe/resource/log/PipeLogStatus.java | 52 +++ .../db/pipe/resource/memory/PipeMemoryBlock.java | 5 + .../db/pipe/resource/memory/PipeMemoryManager.java | 91 +++- .../resource/memory/PipeTabletMemoryBlock.java | 64 +++ .../resource/tsfile/PipeTsFileResourceManager.java | 20 +- .../pipe/resource/wal/PipeWALResourceManager.java | 23 +- .../pipe/task/connection/PipeEventCollector.java | 87 ++++ .../db/pipe/task/subtask/PipeDataNodeSubtask.java | 4 +- .../subtask/connector/PipeConnectorSubtask.java | 12 +- .../subtask/processor/PipeProcessorSubtask.java | 16 +- .../processor/PipeProcessorSubtaskWorker.java | 6 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 6 + .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 151 ++++--- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 10 +- .../execution/aggregation/Accumulator.java | 5 +- .../execution/aggregation/AccumulatorFactory.java | 85 ++-- .../execution/aggregation/Aggregator.java | 4 +- .../execution/aggregation/AvgAccumulator.java | 30 +- .../execution/aggregation/CountAccumulator.java | 14 +- .../execution/aggregation/CountIfAccumulator.java | 9 +- .../aggregation/CountTimeAccumulator.java | 8 +- .../execution/aggregation/ExtremeAccumulator.java | 30 +- .../aggregation/FirstValueAccumulator.java | 44 +- .../aggregation/FirstValueDescAccumulator.java | 30 +- .../aggregation/LastValueAccumulator.java | 44 +- .../aggregation/LastValueDescAccumulator.java | 30 +- .../execution/aggregation/MaxByAccumulator.java | 400 +---------------- ...cumulator.java => MaxMinByBaseAccumulator.java} | 93 ++-- .../execution/aggregation/MaxTimeAccumulator.java | 9 +- .../aggregation/MaxTimeDescAccumulator.java | 9 +- .../execution/aggregation/MaxValueAccumulator.java | 30 +- ...eDescAccumulator.java => MinByAccumulator.java} | 35 +- .../execution/aggregation/MinTimeAccumulator.java | 9 +- .../aggregation/MinTimeDescAccumulator.java | 9 +- .../execution/aggregation/MinValueAccumulator.java | 30 +- .../execution/aggregation/SumAccumulator.java | 30 +- .../aggregation/TimeDurationAccumulator.java | 11 +- .../execution/aggregation/UDAFAccumulator.java | 212 +++++++++ .../execution/aggregation/VarianceAccumulator.java | 30 +- .../SlidingWindowAggregatorFactory.java | 44 +- .../execution/operator/AggregationUtil.java | 8 +- .../process/RawDataAggregationOperator.java | 4 +- .../execution/operator/source/SeriesScanUtil.java | 38 +- .../db/queryengine/plan/analyze/Analysis.java | 11 + .../plan/analyze/ExpressionAnalyzer.java | 6 +- .../plan/analyze/ExpressionTypeAnalyzer.java | 23 +- .../cache/schema/DataNodeDevicePathCache.java | 1 + .../config/executor/ClusterConfigTaskExecutor.java | 32 +- .../config/metadata/ShowClusterIdTask.java | 5 +- .../config/metadata/ShowFunctionsTask.java | 29 +- .../db/queryengine/plan/expression/Expression.java | 8 + .../plan/expression/multi/FunctionExpression.java | 23 +- .../plan/expression/multi/FunctionType.java | 5 +- .../CollectAggregationExpressionsVisitor.java | 2 +- .../visitor/ColumnTransformerVisitor.java | 4 +- .../visitor/IntermediateLayerVisitor.java | 4 +- .../visitor/ReplaceLogicalViewVisitor.java | 2 +- .../cartesian/BindSchemaForExpressionVisitor.java | 4 +- .../cartesian/BindSchemaForPredicateVisitor.java | 4 +- ...catDeviceAndBindSchemaForExpressionVisitor.java | 4 +- ...ncatDeviceAndBindSchemaForPredicateVisitor.java | 2 +- .../db/queryengine/plan/parser/ASTVisitor.java | 107 +++-- .../plan/planner/LogicalPlanBuilder.java | 51 ++- .../plan/planner/LogicalPlanVisitor.java | 5 + .../plan/planner/OperatorTreeGenerator.java | 25 +- .../planner/distribution/DistributionPlanner.java | 2 +- .../planner/distribution/ExchangeNodeAdder.java | 62 +-- .../planner/distribution/NodeDistribution.java | 24 +- .../plan/planner/distribution/SourceRewriter.java | 328 +++++++++----- ...anNodeRewriter.java => BaseSourceRewriter.java} | 2 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 9 + .../plan/planner/plan/node/PlanNode.java | 2 +- .../plan/planner/plan/node/PlanNodeType.java | 7 +- .../plan/planner/plan/node/PlanVisitor.java | 5 + .../node/process/AggregationMergeSortNode.java | 143 ++++++ .../plan/parameter/AggregationDescriptor.java | 18 +- .../plan/statement/crud/QueryStatement.java | 6 +- .../dag/udf/UDAFInformationInferrer.java | 83 ++++ .../mtree/impl/pbtree/CachedMTreeStore.java | 141 +++--- .../impl/pbtree/flush/PBTreeFlushExecutor.java | 9 + .../mtree/impl/pbtree/memory/MemoryManager.java | 57 ++- .../mnode/container/CachedMNodeContainer.java | 147 +++--- .../mnode/container/ICachedMNodeContainer.java | 18 +- .../pbtree/mnode/container/IMNodeChildBuffer.java | 45 ++ .../pbtree/mnode/container/MNodeChildBuffer.java | 298 +++++++++++++ .../mnode/container/MNodeNewChildBuffer.java | 75 ++++ .../mnode/container/MNodeUpdateChildBuffer.java | 83 ++++ .../impl/pbtree/schemafile/MockSchemaFile.java | 6 +- .../pbtree/schemafile/pagemgr/PageManager.java | 16 +- .../schemafile/pagemgr/SchemaPageContext.java | 37 +- .../java/org/apache/iotdb/db/service/DataNode.java | 3 + .../db/storageengine/dataregion/DataRegion.java | 13 +- .../dataregion/wal/utils/WALEntryHandler.java | 5 +- .../org/apache/iotdb/db/utils/DateTimeUtils.java | 17 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 81 +++- .../apache/iotdb/db/utils/TypeInferenceUtils.java | 17 +- .../iotdb/db/utils/constant/SqlConstant.java | 1 + .../iotdb/db/utils/constant/TestConstant.java | 4 + .../org/apache/iotdb/db/auth/entity/RoleTest.java | 31 +- .../metadata/container/MNodeChildBufferTest.java | 125 ++++++ .../mtree/schemafile/SchemaFileLogTest.java | 4 + .../metadata/mtree/schemafile/SchemaFileTest.java | 129 +++--- .../execution/aggregation/AccumulatorTest.java | 105 +++-- .../operator/AggregationOperatorTest.java | 29 +- .../AlignedSeriesAggregationScanOperatorTest.java | 40 +- .../operator/HorizontallyConcatOperatorTest.java | 2 +- .../execution/operator/OperatorMemoryTest.java | 8 +- .../operator/RawDataAggregationOperatorTest.java | 15 +- .../SeriesAggregationScanOperatorTest.java | 43 +- .../SlidingWindowAggregationOperatorTest.java | 9 +- .../distribution/AggregationAlignByDeviceTest.java | 451 +++++++++++++++++++ .../distribution/AggregationDistributionTest.java | 18 +- .../AlignedSeriesScanPredicatePushDownTest.java | 7 +- .../series/SeriesScanLimitOffsetPushDownTest.java | 5 + .../series/SeriesScanPredicatePushDownTest.java | 23 +- .../iotdb/db/tools/PBTreeFileSketchTest.java | 5 + .../apache/iotdb/db/utils/DateTimeUtilsTest.java | 44 ++ .../apache/iotdb/db/utils/EnvironmentUtils.java | 21 +- .../resources/conf/iotdb-common.properties | 9 +- .../commons/auth/authorizer/BasicAuthorizer.java | 10 + .../org/apache/iotdb/commons/auth/entity/Role.java | 14 +- .../commons/auth/role/LocalFileRoleAccessor.java | 11 + .../iotdb/commons/auth/user/BasicUserManager.java | 15 +- .../commons/auth/user/LocalFileUserAccessor.java | 11 + .../apache/iotdb/commons/conf/CommonConfig.java | 109 ++++- .../iotdb/commons/conf/CommonDescriptor.java | 37 ++ .../apache/iotdb/commons/conf/IoTDBConstant.java | 3 +- .../commons/pipe/agent/task/PipeTaskAgent.java | 39 +- .../iotdb/commons/pipe/config/PipeConfig.java | 40 ++ .../commons/pipe/task/meta/PipeMetaKeeper.java | 4 + .../commons/pipe/task/subtask/PipeSubtask.java | 4 - .../iotdb/commons/schema/MergeSortIterator.java | 122 +++++ .../udf/builtin/BuiltinAggregationFunction.java | 5 +- .../commons/udf/service/UDFManagementService.java | 49 +- .../org/apache/iotdb/commons/utils/FileUtils.java | 76 ++++ .../org/apache/iotdb/commons/utils/TestOnly.java | 8 +- .../tsfile/read/common/block/TsBlockUtil.java | 55 +++ .../tsfile/read/reader/page/AlignedPageReader.java | 54 +-- .../thrift-commons/src/main/thrift/common.thrift | 2 + .../src/main/thrift/confignode.thrift | 4 + pom.xml | 4 +- 264 files changed, 10556 insertions(+), 2870 deletions(-)
