This is an automated email from the ASF dual-hosted git repository. chenyz pushed a commit to branch pbtree_page_concurrency in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6c6e66f39d10d6d36bc0547b87e7cd4f38424821 Merge: 035c224d6fd 2c8a8b0346f Author: Chen YZ <[email protected]> AuthorDate: Wed Mar 6 11:26:18 2024 +0800 resolve conflict .github/workflows/client-cpp.yml | 10 +- .github/workflows/client-go.yml | 6 +- .github/workflows/client-python.yml | 4 +- .github/workflows/cluster-it-1c1d.yml | 8 +- .github/workflows/cluster-it-1c3d.yml | 22 +- .github/workflows/grafana-plugin.yml | 10 +- .github/workflows/greetings.yml | 2 +- .github/workflows/pipe-it-2cluster.yml | 6 +- .github/workflows/sonar-codecov.yml | 12 +- .github/workflows/unit-test.yml | 6 +- NOTICE | 2 +- NOTICE-binary | 2 +- example/flink-sql/pom.xml | 2 +- .../java/org/apache/iotdb/udf/UDAFExample.java | 198 +++ integration-test/pom.xml | 14 +- .../apache/iotdb/db/query/udf/example/UDAFAvg.java | 190 +++ .../iotdb/db/query/udf/example/UDAFCount.java | 112 ++ .../apache/iotdb/db/query/udf/example/UDAFSum.java | 189 +++ .../it/env/cluster/config/MppCommonConfig.java | 6 + .../env/cluster/config/MppSharedCommonConfig.java | 7 + .../iotdb/it/env/cluster/env/AbstractEnv.java | 188 +-- .../it/env/remote/config/RemoteCommonConfig.java | 5 + .../iotdb/it/env/remote/env/RemoteServerEnv.java | 20 + .../constant/BuiltinAggregationFunctionEnum.java | 3 +- .../apache/iotdb/itbase/constant/TestConstant.java | 12 + .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 24 +- .../org/apache/iotdb/itbase/env/CommonConfig.java | 2 + .../it/cluster/IoTDBClusterRestartIT.java | 42 +- .../confignode/it/cluster/IoTDBClusterStartIT.java | 28 +- .../confignode/it/procedure/IoTDBProcedureIT.java | 137 +++ .../db/it/IOTDBInsertWithTimeAtAnyIndexIT.java | 99 ++ .../org/apache/iotdb/db/it/IoTDBDeletionIT.java | 51 +- .../java/org/apache/iotdb/db/it/IoTDBFilterIT.java | 34 + ...ushQueryMergeIT.java => IoTDBFlushQueryIT.java} | 3 +- .../apache/iotdb/db/it/IoTDBInsertMultiRowIT.java | 1 - .../iotdb/db/it/IoTDBInsertWithoutTimeIT.java | 8 +- .../org/apache/iotdb/db/it/IoTDBMultiDeviceIT.java | 1 - .../org/apache/iotdb/db/it/IoTDBRecoverIT.java | 307 ++--- .../apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java | 166 ++- .../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 4 +- .../org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java | 20 - .../iotdb/db/it/IoTDBSimpleQueryStandaloneIT.java | 75 ++ .../it/IoTDBSyntaxConventionStringLiteralIT.java | 8 +- .../db/it/aggregation/IoTDBAggregationIT.java | 104 ++ .../aggregation/IoTDBCountTimeAlignedDeviceIT.java | 2 +- .../db/it/aggregation/maxby/IoTDBMaxBy2IT.java | 25 +- .../db/it/aggregation/maxby/IoTDBMaxBy3IT.java | 57 +- .../maxby/IoTDBMaxByAlignedSeriesIT.java | 71 ++ .../db/it/aggregation/maxby/IoTDBMaxByIT.java | 486 ++++++++ .../db/it/aggregation/minby/IoTDBMinBy2IT.java | 25 +- .../db/it/aggregation/minby/IoTDBMinBy3IT.java | 57 +- .../minby/IoTDBMinByAlignedSeriesIT.java | 71 ++ .../db/it/aggregation/minby/IoTDBMinByIT.java | 486 ++++++++ .../IoTDBOrderByLimitOffsetAlignByDeviceIT.java | 1 + .../db/it/aligned/IoTDBAlignedDataDeletionIT.java | 52 +- .../db/it/aligned/IoTDBAlignedSeriesQueryIT.java | 48 + .../db/it/aligned/IoTDBPredicatePushDown2IT.java | 55 + .../db/it/aligned/IoTDBPredicatePushDown3IT.java | 55 + .../db/it/aligned/IoTDBPredicatePushDown4IT.java | 55 + .../db/it/aligned/IoTDBPredicatePushDownIT.java | 601 +++++++++ .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 2 + .../iotdb/db/it/auth/IoTDBSystemPermissionIT.java | 2 - .../org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java | 10 +- .../apache/iotdb/db/it/cq/IoTDBCQExecInNsIT.java | 10 +- .../apache/iotdb/db/it/cq/IoTDBCQExecInUsIT.java | 10 +- .../db/it/udaf/IoTDBUDAFGroupByConditionIT.java | 325 +++++ .../iotdb/db/it/udaf/IoTDBUDAFGroupByCountIT.java | 292 +++++ .../iotdb/db/it/udaf/IoTDBUDAFGroupByLevelIT.java | 495 ++++++++ .../db/it/udaf/IoTDBUDAFGroupBySessionIT.java | 302 +++++ .../iotdb/db/it/udaf/IoTDBUDAFGroupByTimeIT.java | 340 +++++ .../db/it/udaf/IoTDBUDAFGroupByVariationIT.java | 494 ++++++++ .../apache/iotdb/db/it/udaf/IoTDBUDAFHavingIT.java | 287 +++++ .../iotdb/db/it/udaf/IoTDBUDAFManagementIT.java | 336 +++++ .../apache/iotdb/db/it/udaf/IoTDBUDAFMiscIT.java | 126 ++ .../iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java | 440 +++++++ .../iotdb/db/it/udaf/IoTDBUDAFOrderByIT.java | 468 +++++++ .../org/apache/iotdb/db/it/utils/TestUtils.java | 21 +- .../org/apache/iotdb/pipe/it/IoTDBPipeAlterIT.java | 355 ++++++ .../apache/iotdb/pipe/it/IoTDBPipeClusterIT.java | 38 +- .../apache/iotdb/pipe/it/IoTDBPipeExtractorIT.java | 37 +- .../apache/iotdb/pipe/it/IoTDBPipeLifeCycleIT.java | 25 +- .../apache/iotdb/pipe/it/IoTDBPipeSyntaxIT.java | 3 +- .../iotdb/session/it/IoTDBSessionComplexIT.java | 2 - .../test/java/org/apache/iotdb/spark/it/Utils.java | 1 - .../java/org/apache/iotdb/tsfile/utils/BitMap.java | 0 .../java/org/apache/iotdb/pipe/api/PipeSink.java | 49 +- .../java/org/apache/iotdb/pipe/api/PipeSource.java | 42 +- .../api/customizer/parameter/PipeParameters.java | 23 + .../iotdb/pipe/api/event/UserDefinedEvent.java | 58 + .../main/java/org/apache/iotdb/udf/api/State.java | 28 +- .../main/java/org/apache/iotdb/udf/api/UDAF.java | 71 ++ .../api/customizer/config/UDAFConfigurations.java | 15 +- .../apache/iotdb/udf/api/utils/ResultValue.java | 76 ++ .../cli/src/assembly/resources/sbin/start-cli.bat | 2 + .../src/assembly/resources/tools/collect-info.bat | 2 + .../src/assembly/resources/tools/export-csv.bat | 3 + .../src/assembly/resources/tools/export-tsfile.bat | 3 + .../src/assembly/resources/tools/import-csv.bat | 3 + .../src/assembly/resources/tools/load-tsfile.bat | 2 + .../java/org/apache/iotdb/cli/AbstractCli.java | 246 ++-- .../src/main/java/org/apache/iotdb/cli/Cli.java | 124 +- .../apache/iotdb/cli/IoTDBSyntaxHighlighter.java | 5 +- .../java/org/apache/iotdb/cli/type/ExitType.java | 10 +- .../org/apache/iotdb/cli/utils/CliContext.java | 84 ++ .../org/apache/iotdb/cli/utils/IoTPrinter.java | 36 +- .../org/apache/iotdb/cli/utils/JlineUtils.java | 12 +- .../org/apache/iotdb/tool/AbstractCsvTool.java | 10 +- .../org/apache/iotdb/tool/AbstractTsFileTool.java | 6 +- .../main/java/org/apache/iotdb/tool/ExportCsv.java | 33 +- .../java/org/apache/iotdb/tool/ExportTsFile.java | 39 +- .../main/java/org/apache/iotdb/tool/ImportCsv.java | 48 +- .../java/org/apache/iotdb/cli/AbstractCliIT.java | 37 +- .../iotdb/tool/integration/ExportCsvTestIT.java | 3 + .../iotdb/tool/integration/ImportCsvTestIT.java | 3 + .../org/apache/iotdb/isession/SessionConfig.java | 4 + iotdb-client/jdbc/pom.xml | 4 + .../org/apache/iotdb/jdbc/IoTDBConnection.java | 4 +- .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 39 +- .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 9 + .../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 4 +- .../java/org/apache/iotdb/session/Session.java | 27 +- .../apache/iotdb/session/SessionConnection.java | 660 ++++++++-- .../org/apache/iotdb/session/pool/SessionPool.java | 1295 +++++++++----------- .../iotdb/session/SessionConnectionTest.java | 9 +- .../sql/factory/IoTDBDynamicTableFactory.java | 2 + iotdb-connector/grafana-plugin/yarn.lock | 6 +- iotdb-core/antlr/pom.xml | 11 + .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 2 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 73 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 8 + .../assembly/resources/sbin/start-confignode.bat | 4 + .../confignode/client/DataNodeRequestType.java | 3 +- .../client/async/AsyncDataNodeClientPool.java | 9 +- .../client/async/handlers/AsyncClientHandler.java | 3 +- .../confignode/conf/SystemPropertiesUtils.java | 11 +- .../consensus/request/ConfigPhysicalPlan.java | 4 + .../consensus/request/ConfigPhysicalPlanType.java | 1 + .../request/write/pipe/task/AlterPipePlanV2.java | 66 + .../statemachine/ConfigRegionStateMachine.java | 18 +- .../iotdb/confignode/manager/ClusterManager.java | 13 + .../iotdb/confignode/manager/ConfigManager.java | 29 +- .../apache/iotdb/confignode/manager/IManager.java | 23 +- .../iotdb/confignode/manager/ProcedureManager.java | 32 +- .../manager/consensus/ConsensusManager.java | 58 +- .../iotdb/confignode/manager/cq/CQManager.java | 11 - .../iotdb/confignode/manager/load/LoadManager.java | 11 + .../manager/load/balancer/PartitionBalancer.java | 31 +- .../manager/load/balancer/RouteBalancer.java | 37 +- .../confignode/manager/load/cache/LoadCache.java | 7 +- .../manager/load/cache/route/RegionRouteCache.java | 5 + .../manager/load/service/StatisticsService.java | 16 +- .../iotdb/confignode/manager/node/NodeManager.java | 49 +- .../pipe/coordinator/runtime/PipeMetaSyncer.java | 10 - .../pipe/coordinator/task/PipeTaskCoordinator.java | 10 + .../confignode/persistence/ProcedureInfo.java | 7 +- .../persistence/executor/ConfigPlanExecutor.java | 3 + .../persistence/pipe/PipePluginInfo.java | 22 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 100 +- .../iotdb/confignode/procedure/Procedure.java | 15 + .../confignode/procedure/ProcedureExecutor.java | 2 + .../procedure/env/ConfigNodeProcedureEnv.java | 2 + .../impl/CreateManyDatabasesProcedure.java | 116 ++ .../impl/pipe/AbstractOperatePipeProcedureV2.java | 21 +- .../procedure/impl/pipe/PipeTaskOperation.java | 1 + .../impl/pipe/task/AlterPipeProcedureV2.java | 354 ++++++ .../impl/pipe/task/CreatePipeProcedureV2.java | 25 +- .../impl/pipe/task/StopPipeProcedureV2.java | 3 +- .../procedure/store/ConfigProcedureStore.java | 9 + .../procedure/store/ProcedureFactory.java | 15 +- .../confignode/procedure/store/ProcedureStore.java | 192 --- .../confignode/procedure/store/ProcedureType.java | 6 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 20 +- .../request/ConfigPhysicalPlanSerDeTest.java | 29 +- .../iotdb/confignode/persistence/PipeInfoTest.java | 2 +- ...TestSTMProcedure.java => STMProcedureTest.java} | 2 +- .../impl/pipe/task/AlterPipeProcedureV2Test.java | 65 + .../procedure/store/TestProcedureStore.java | 113 -- .../exception/RatisReadUnavailableException.java | 2 +- .../resources/conf/iotdb-datanode.properties | 6 +- .../src/assembly/resources/sbin/start-datanode.bat | 5 + .../src/main/codegen/templates/ModeAccumulator.ftl | 5 +- .../org/apache/iotdb/db/audit/AuditLogger.java | 5 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 116 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 182 ++- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 7 +- .../db/consensus/DataRegionConsensusImpl.java | 294 ++--- .../db/consensus/SchemaRegionConsensusImpl.java | 239 ++-- .../dataregion/DataRegionStateMachine.java | 10 +- .../IoTConsensusDataRegionStateMachine.java | 2 +- .../schemaregion/SchemaExecutionVisitor.java | 1 - .../pipe/agent/plugin/PipePluginDataNodeAgent.java | 13 + .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 6 +- .../db/pipe/agent/task/PipeTaskDataNodeAgent.java | 30 +- .../common/PipeTransferHandshakeConstant.java} | 15 +- ...akeReq.java => PipeTransferHandshakeV1Req.java} | 18 +- ...akeReq.java => PipeTransferHandshakeV2Req.java} | 63 +- .../protocol/airgap/IoTDBAirGapConnector.java | 58 +- .../async/IoTDBThriftAsyncClientManager.java | 61 +- .../thrift/async/IoTDBThriftAsyncConnector.java | 46 +- .../PipeTransferTabletBatchEventHandler.java | 9 +- .../PipeTransferTabletInsertionEventHandler.java | 4 +- .../thrift/sync/IoTDBThriftSyncClientManager.java | 58 +- .../thrift/sync/IoTDBThriftSyncConnector.java | 34 - .../protocol/writeback/WriteBackConnector.java | 22 +- .../apache/iotdb/db/pipe/event/EnrichedEvent.java | 46 +- .../db/pipe/event/UserDefinedEnrichedEvent.java | 86 ++ .../tablet/PipeInsertNodeTabletInsertionEvent.java | 18 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 25 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 31 +- .../tsfile/TsFileInsertionDataContainer.java | 16 + .../pipe/extractor/IoTDBDataRegionExtractor.java | 13 +- .../PipeHistoricalDataRegionTsFileExtractor.java | 41 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 9 +- .../PipeRealtimeDataRegionHybridExtractor.java | 22 +- .../PipeRealtimeDataRegionLogExtractor.java | 12 +- .../pipe/extractor/realtime/epoch/TsFileEpoch.java | 14 + .../realtime/epoch/TsFileEpochManager.java | 5 +- .../legacy/IoTDBLegacyPipeReceiverAgent.java | 2 +- .../receiver/legacy/loader/DeletionLoader.java | 2 +- .../pipe/receiver/legacy/loader/TsFileLoader.java | 2 +- .../receiver/thrift/IoTDBThriftReceiverV1.java | 59 +- .../db/pipe/resource/PipeResourceManager.java | 7 + .../db/pipe/resource/log/PipeLogManager.java} | 26 +- .../iotdb/db/pipe/resource/log/PipeLogStatus.java | 52 + .../db/pipe/resource/memory/PipeMemoryBlock.java | 5 + .../db/pipe/resource/memory/PipeMemoryManager.java | 91 +- .../resource/memory/PipeTabletMemoryBlock.java | 64 + .../resource/tsfile/PipeTsFileResourceManager.java | 20 +- .../pipe/resource/wal/PipeWALResourceManager.java | 23 +- .../pipe/task/builder/PipeDataNodeTaskBuilder.java | 31 +- .../pipe/task/connection/PipeEventCollector.java | 87 ++ .../db/pipe/task/subtask/PipeDataNodeSubtask.java | 4 +- .../subtask/connector/PipeConnectorSubtask.java | 33 +- .../connector/PipeConnectorSubtaskLifeCycle.java | 42 +- .../subtask/processor/PipeProcessorSubtask.java | 28 +- .../processor/PipeProcessorSubtaskWorker.java | 6 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 60 +- .../protocol/client/ConfigNodeClientManager.java | 9 +- .../iotdb/db/protocol/client/ConfigNodeInfo.java | 6 +- .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 151 +-- .../protocol/rest/filter/AuthorizationFilter.java | 2 +- .../iotdb/db/protocol/session/SessionManager.java | 11 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 19 +- .../impl/DataNodeInternalRPCServiceImpl.java | 18 +- .../db/queryengine/common/MPPQueryContext.java | 5 + .../iotdb/db/queryengine/common/SessionInfo.java | 14 +- .../common/header/ColumnHeaderConstant.java | 6 + .../common/header/DatasetHeaderFactory.java | 4 + .../execution/aggregation/Accumulator.java | 5 +- .../execution/aggregation/AccumulatorFactory.java | 68 +- .../execution/aggregation/Aggregator.java | 22 +- .../execution/aggregation/AvgAccumulator.java | 30 +- .../execution/aggregation/CountAccumulator.java | 14 +- .../execution/aggregation/CountIfAccumulator.java | 9 +- .../aggregation/CountTimeAccumulator.java | 8 +- .../execution/aggregation/ExtremeAccumulator.java | 30 +- .../aggregation/FirstValueAccumulator.java | 44 +- .../aggregation/FirstValueDescAccumulator.java | 30 +- .../aggregation/LastValueAccumulator.java | 44 +- .../aggregation/LastValueDescAccumulator.java | 30 +- ...eDescAccumulator.java => MaxByAccumulator.java} | 38 +- .../aggregation/MaxMinByBaseAccumulator.java | 441 +++++++ .../execution/aggregation/MaxTimeAccumulator.java | 9 +- .../aggregation/MaxTimeDescAccumulator.java | 9 +- .../execution/aggregation/MaxValueAccumulator.java | 30 +- ...eDescAccumulator.java => MinByAccumulator.java} | 38 +- .../execution/aggregation/MinTimeAccumulator.java | 9 +- .../aggregation/MinTimeDescAccumulator.java | 9 +- .../execution/aggregation/MinValueAccumulator.java | 30 +- .../execution/aggregation/SumAccumulator.java | 30 +- .../aggregation/TimeDurationAccumulator.java | 11 +- .../execution/aggregation/UDAFAccumulator.java | 212 ++++ .../execution/aggregation/VarianceAccumulator.java | 30 +- .../SlidingWindowAggregatorFactory.java | 77 +- .../execution/executor/RegionExecutionResult.java | 9 + .../execution/executor/RegionReadExecutor.java | 23 +- .../fragment/FragmentInstanceContext.java | 2 +- .../execution/fragment/QueryContext.java | 26 +- .../execution/load/LoadTsFileManager.java | 35 +- .../execution/operator/AggregationUtil.java | 8 +- .../operator/process/ProjectOperator.java | 101 ++ .../process/RawDataAggregationOperator.java | 4 +- .../execution/operator/process/TopKOperator.java | 44 +- .../AbstractSeriesAggregationScanOperator.java | 2 +- .../operator/source/AlignedSeriesScanOperator.java | 23 +- .../operator/source/SeriesScanOperator.java | 3 +- .../execution/operator/source/SeriesScanUtil.java | 38 +- .../execution/schedule/DriverScheduler.java | 2 +- .../iotdb/db/queryengine/plan/Coordinator.java | 11 +- .../db/queryengine/plan/analyze/Analysis.java | 26 + .../queryengine/plan/analyze/AnalyzeVisitor.java | 119 +- .../plan/analyze/ExpressionAnalyzer.java | 6 +- .../plan/analyze/ExpressionTypeAnalyzer.java | 72 +- .../queryengine/plan/analyze/ExpressionUtils.java | 11 + .../queryengine/plan/analyze/PredicateUtils.java | 157 ++- .../queryengine/plan/analyze/TemplatedAnalyze.java | 20 +- .../db/queryengine/plan/analyze/TemplatedInfo.java | 169 ++- .../db/queryengine/plan/analyze/TypeProvider.java | 4 + .../cache/schema/DataNodeDevicePathCache.java | 20 +- .../queryengine/plan/execution/QueryExecution.java | 6 +- .../plan/execution/config/ConfigTaskVisitor.java | 26 +- .../config/executor/ClusterConfigTaskExecutor.java | 123 +- .../config/executor/IConfigTaskExecutor.java | 7 +- .../config/metadata/ShowClusterIdTask.java | 5 +- .../config/metadata/ShowFunctionsTask.java | 29 +- ...epairDataTask.java => StartRepairDataTask.java} | 12 +- ...RepairDataTask.java => StopRepairDataTask.java} | 12 +- .../AlterPipeTask.java} | 14 +- .../memory/StatementMemorySourceVisitor.java | 24 +- .../db/queryengine/plan/expression/Expression.java | 8 + .../plan/expression/ExpressionFactory.java | 35 + .../plan/expression/leaf/ConstantOperand.java | 15 +- .../plan/expression/multi/FunctionExpression.java | 23 +- .../plan/expression/multi/FunctionType.java | 5 +- .../plan/expression/ternary/BetweenExpression.java | 6 +- .../BindTypeForTimeSeriesOperandVisitor.java | 13 - .../CollectAggregationExpressionsVisitor.java | 2 +- .../visitor/ColumnTransformerVisitor.java | 4 +- .../visitor/ExpressionNormalizeVisitor.java | 13 +- .../visitor/GetMeasurementExpressionVisitor.java | 18 - .../visitor/IntermediateLayerVisitor.java | 4 +- .../visitor/LowercaseNormalizeVisitor.java | 13 +- .../expression/visitor/ReconstructVisitor.java | 33 + .../visitor/ReplaceLogicalViewVisitor.java | 2 +- .../ReplaceRawPathWithGroupedPathVisitor.java | 24 - .../visitor/ReplaceSubTreeWithViewVisitor.java | 16 - .../cartesian/BindSchemaForExpressionVisitor.java | 11 +- .../cartesian/BindSchemaForPredicateVisitor.java | 10 +- ...catDeviceAndBindSchemaForExpressionVisitor.java | 11 +- ...ncatDeviceAndBindSchemaForPredicateVisitor.java | 2 +- .../ConcatExpressionWithSuffixPathsVisitor.java | 23 +- .../visitor/logical/LogicalAndVisitor.java | 70 ++ .../visitor/logical/LogicalOrVisitor.java | 70 ++ .../PredicateCanPushDownToSourceChecker.java} | 21 +- .../visitor/logical/TimeFilterExistChecker.java} | 17 +- .../predicate/ConvertPredicateToFilterVisitor.java | 406 ++++++ .../ConvertPredicateToTimeFilterVisitor.java | 30 +- .../predicate/PredicatePushIntoScanChecker.java | 171 +++ .../visitor/predicate/PredicateSimplifier.java | 279 +++++ .../plan/optimization/PredicatePushDown.java | 468 +++++++ .../db/queryengine/plan/parser/ASTVisitor.java | 258 ++-- .../plan/parser/StatementGenerator.java | 13 +- .../plan/planner/LocalExecutionPlanContext.java | 14 + .../plan/planner/LocalExecutionPlanner.java | 4 +- .../plan/planner/LogicalPlanBuilder.java | 150 ++- .../plan/planner/LogicalPlanVisitor.java | 20 +- .../queryengine/plan/planner/LogicalPlanner.java | 7 +- .../plan/planner/OperatorTreeGenerator.java | 470 +++---- .../plan/planner/SubPlanTypeExtractor.java | 11 + .../plan/planner/TemplatedLogicalPlan.java | 8 +- .../plan/planner/TemplatedLogicalPlanBuilder.java | 4 +- .../distribution/DistributionPlanContext.java | 7 +- .../planner/distribution/DistributionPlanner.java | 2 +- .../planner/distribution/ExchangeNodeAdder.java | 68 +- .../planner/distribution/NodeDistribution.java | 24 +- .../plan/planner/distribution/SourceRewriter.java | 398 +++--- ...anNodeRewriter.java => BaseSourceRewriter.java} | 2 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 58 +- .../plan/planner/plan/node/PlanNode.java | 2 +- .../plan/planner/plan/node/PlanNodeType.java | 9 +- .../plan/planner/plan/node/PlanVisitor.java | 14 +- .../node/process/AggregationMergeSortNode.java | 143 +++ .../plan/planner/plan/node/process/FilterNode.java | 16 +- .../planner/plan/node/process/ProjectNode.java | 25 +- .../planner/plan/node/process/TransformNode.java | 25 +- .../plan/node/process/join/InnerTimeJoinNode.java | 24 +- .../node/process/join/LeftOuterTimeJoinNode.java | 4 +- .../plan/node/source/AlignedSeriesScanNode.java | 128 +- .../planner/plan/node/source/SeriesScanNode.java | 119 +- ...riesScanNode.java => SeriesScanSourceNode.java} | 163 +-- .../planner/plan/node/write/DeleteDataNode.java | 5 +- .../planner/plan/node/write/InsertRowNode.java | 5 +- .../planner/plan/node/write/InsertTabletNode.java | 4 +- .../plan/parameter/AggregationDescriptor.java | 54 +- .../CrossSeriesAggregationDescriptor.java | 146 ++- .../plan/scheduler/AsyncSendPlanNodeHandler.java | 4 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 42 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 33 +- .../queryengine/plan/statement/StatementType.java | 4 +- .../plan/statement/StatementVisitor.java | 22 +- .../plan/statement/component/SelectComponent.java | 11 +- .../plan/statement/crud/QueryStatement.java | 6 +- .../metadata/CreateContinuousQueryStatement.java | 4 - .../metadata/ShowCurrentTimestampStatement.java} | 12 +- .../metadata/pipe/AlterPipeStatement.java | 113 ++ .../plan/statement/sys/ShowQueriesStatement.java | 11 - ...tatement.java => StartRepairDataStatement.java} | 6 +- ...Statement.java => StopRepairDataStatement.java} | 6 +- .../dag/udf/UDAFInformationInferrer.java | 83 ++ .../schemaregion/SchemaRegionLoader.java | 14 + .../schemaregion/impl/SchemaRegionMemoryImpl.java | 1 + .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 1 + .../mtree/impl/pbtree/CachedMTreeStore.java | 143 +-- .../impl/pbtree/flush/PBTreeFlushExecutor.java | 9 + .../mtree/impl/pbtree/memory/MemoryManager.java | 59 +- .../mnode/container/CachedMNodeContainer.java | 147 +-- .../mnode/container/ICachedMNodeContainer.java | 18 +- .../pbtree/mnode/container/IMNodeChildBuffer.java | 45 + .../pbtree/mnode/container/MNodeChildBuffer.java | 298 +++++ .../mnode/container/MNodeNewChildBuffer.java | 75 ++ .../mnode/container/MNodeUpdateChildBuffer.java | 83 ++ .../impl/pbtree/schemafile/AliasIndexPage.java | 3 +- .../mtree/impl/pbtree/schemafile/ISegment.java | 7 +- .../impl/pbtree/schemafile/ISegmentedPage.java | 11 +- .../mtree/impl/pbtree/schemafile/InternalPage.java | 3 +- .../impl/pbtree/schemafile/MockSchemaFile.java | 6 +- .../impl/pbtree/schemafile/SegmentedPage.java | 43 +- .../impl/pbtree/schemafile/WrappedSegment.java | 10 +- .../pbtree/schemafile/pagemgr/PageIOChannel.java | 2 +- .../pbtree/schemafile/pagemgr/PageManager.java | 61 +- .../schemafile/pagemgr/SchemaPageContext.java | 37 +- .../java/org/apache/iotdb/db/service/DataNode.java | 19 +- .../apache/iotdb/db/service/IoTDBShutdownHook.java | 5 + .../metrics/IoTDBInternalLocalReporter.java | 2 +- .../iotdb/db/storageengine/StorageEngine.java | 48 +- .../db/storageengine/dataregion/DataRegion.java | 457 +++---- .../dataregion/DeviceLastFlushTime.java} | 38 +- .../dataregion/HashLastFlushTimeMap.java | 92 +- .../dataregion/ILastFlushTime.java} | 12 +- .../dataregion/ILastFlushTimeMap.java | 12 +- .../dataregion/PartitionLastFlushTime.java} | 31 +- .../CompactionValidationFailedException.java | 15 + .../impl/ReadChunkCompactionPerformer.java | 1 - .../execute/task/AbstractCompactionTask.java | 62 +- .../execute/task/InnerSpaceCompactionTask.java | 11 +- .../task/InsertionCrossSpaceCompactionTask.java | 16 +- .../writer/AbstractCrossCompactionWriter.java | 9 +- .../writer/AbstractInnerCompactionWriter.java | 8 - .../writer/RepairUnsortedFileCompactionWriter.java | 10 +- .../compaction/io/CompactionTsFileWriter.java | 5 +- .../compaction/repair/RepairDataFileScanUtil.java | 271 ++++ .../dataregion/compaction/repair/RepairLogger.java | 146 ++- .../RepairProgress.java} | 23 +- .../repair/RepairTaskRecoverLogParser.java | 30 +- .../compaction/repair/RepairTaskStatus.java} | 11 +- .../compaction/repair/RepairTimePartition.java | 4 + .../repair/RepairTimePartitionScanTask.java | 202 +++ .../repair/UnsortedFileRepairTaskScheduler.java | 316 ++--- .../schedule/CompactionScheduleTaskManager.java | 317 +++++ .../schedule/CompactionScheduleTaskWorker.java | 69 ++ .../compaction/schedule/CompactionScheduler.java | 101 +- .../compaction/schedule/CompactionTaskManager.java | 6 +- .../compaction/schedule/CompactionTaskQueue.java | 101 ++ .../compaction/schedule/CompactionWorker.java | 68 +- .../estimator/AbstractCompactionEstimator.java | 28 +- .../estimator/AbstractCrossSpaceEstimator.java | 6 +- .../estimator/AbstractInnerSpaceEstimator.java | 19 +- .../dataregion/flush/MemTableFlushTask.java | 37 +- .../dataregion/memtable/AbstractMemTable.java | 14 +- .../memtable/AlignedWritableMemChunk.java | 6 +- .../dataregion/memtable/PrimitiveMemTable.java | 5 - .../dataregion/memtable/TsFileProcessor.java | 159 +-- .../dataregion/memtable/WritableMemChunk.java | 2 +- .../dataregion/modification/Deletion.java | 6 +- .../dataregion/tsfile/TsFileManager.java | 36 - .../dataregion/tsfile/TsFileResource.java | 1 + .../tsfile/timeindex/DeviceTimeIndex.java | 14 +- .../dataregion/utils/TsFileResourceUtils.java | 2 +- .../storageengine/dataregion/wal/node/WALNode.java | 32 +- .../dataregion/wal/utils/WALEntryHandler.java | 5 +- .../rescon/memory/MemTableManager.java | 60 +- .../db/storageengine/rescon/memory/SystemInfo.java | 116 +- .../rescon/memory/TimePartitionInfo.java | 10 +- .../rescon/memory/TimePartitionManager.java | 31 +- .../org/apache/iotdb/db/utils/DateTimeUtils.java | 25 +- .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 32 +- .../java/org/apache/iotdb/db/utils/MemUtils.java | 68 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 85 +- .../apache/iotdb/db/utils/TypeInferenceUtils.java | 20 +- .../iotdb/db/utils/constant/SqlConstant.java | 2 + .../iotdb/db/utils/constant/TestConstant.java | 8 + .../db/utils/datastructure/AlignedTVList.java | 4 +- .../iotdb/db/utils/datastructure/BinaryTVList.java | 2 +- .../datastructure/FixedPriorityBlockingQueue.java | 12 +- .../iotdb/db/utils/datastructure/TVList.java | 8 +- .../org/apache/iotdb/db/auth/entity/RoleTest.java | 31 +- .../metadata/container/MNodeChildBufferTest.java | 125 ++ .../mtree/schemafile/SchemaFileLogTest.java | 4 + .../metadata/mtree/schemafile/SchemaFileTest.java | 129 +- .../SchemaRegionSimpleRecoverTest.java | 98 ++ .../connector/PipeDataNodeThriftRequestTest.java | 68 +- .../iotdb/db/pipe/connector/PipeReceiverTest.java | 4 +- .../execution/aggregation/AccumulatorTest.java | 182 ++- .../operator/AggregationOperatorTest.java | 29 +- .../AlignedSeriesAggregationScanOperatorTest.java | 62 +- .../operator/AlignedSeriesScanOperatorTest.java | 15 +- .../operator/HorizontallyConcatOperatorTest.java | 2 +- .../execution/operator/OperatorMemoryTest.java | 38 +- .../operator/RawDataAggregationOperatorTest.java | 15 +- .../SeriesAggregationScanOperatorTest.java | 43 +- .../SlidingWindowAggregationOperatorTest.java | 11 +- .../predicate/PredicatePushIntoScanTest.java | 140 +++ .../predicate/PredicateRemoveNotTest.java | 2 +- .../expression/predicate/PredicateUtilsTest.java | 77 ++ .../plan/analyze/AggregationDescriptorTest.java | 8 +- .../db/queryengine/plan/analyze/AnalyzeTest.java | 10 +- .../optimization/ColumnInjectionPushDownTest.java | 51 +- .../plan/optimization/LimitOffsetPushDownTest.java | 38 +- .../plan/optimization/OptimizationTestUtil.java | 71 +- .../plan/optimization/PredicatePushDownTest.java | 449 +++++++ .../plan/optimization/TestPlanBuilder.java | 94 +- .../plan/parser/StatementGeneratorTest.java | 6 +- .../plan/planner/FragmentInstanceSerdeTest.java | 3 +- .../queryengine/plan/planner/QueryPlannerTest.java | 106 -- .../distribution/AggregationAlignByDeviceTest.java | 465 +++++++ .../distribution/AggregationDistributionTest.java | 49 +- .../AlignByDeviceOrderByLimitOffsetTest.java | 242 ++-- .../AlignByTimeOrderByLimitOffsetTest.java | 5 +- .../planner/distribution/AlignedByDeviceTest.java | 375 +++++- .../plan/planner/distribution/Util.java | 2 +- .../plan/planner/distribution/Util2.java | 2 +- .../DataQueryLogicalPlannerTest.java} | 244 ++-- .../logical/LogicalPlannerTestUtil.java} | 53 +- .../SchemaQueryLogicalPlannerTest.java} | 118 +- .../planner/node/process/FilterNodeSerdeTest.java | 2 - .../node/process/GroupByLevelNodeSerdeTest.java | 3 +- .../node/process/GroupByTagNodeSerdeTest.java | 22 +- .../node/process/TimeJoinNodeSerdeTest.java | 19 +- .../builder/EvaluationDAGBuilderTest.java | 3 +- .../db/storageengine/DevicePathCacheTests.java | 67 + .../dataregion/LastFlushTimeMapTest.java | 121 +- .../compaction/CompactionOverlapCheckTest.java | 14 + .../compaction/CompactionSchedulerTest.java | 1 - .../compaction/CompactionWorkerTest.java | 103 +- .../FastCrossCompactionPerformerTest.java | 22 +- .../cross/CrossSpaceCompactionSelectorTest.java | 38 +- .../cross/InsertionCrossSpaceCompactionTest.java | 13 +- .../compaction/inner/InnerCompactionLogTest.java | 2 +- .../inner/InnerCompactionMoreDataTest.java | 3 +- .../SizeTieredCompactionSelectorTest.java | 38 - .../compaction/repair/AbstractRepairDataTest.java | 66 + .../repair/RepairDataFileScanUtilTest.java | 138 +++ .../compaction/repair/RepairLoggerTest.java | 73 +- .../repair/RepairUnsortedFileCompactionTest.java | 116 +- .../compaction/utils/CompactionConfigRestorer.java | 4 - .../compaction/utils/CompactionTaskQueueTest.java | 201 +++ .../AlignedSeriesScanPredicatePushDownTest.java | 7 +- .../series/SeriesScanLimitOffsetPushDownTest.java | 5 + .../series/SeriesScanPredicatePushDownTest.java | 23 +- .../rescon/memory/TimePartitionManagerTest.java | 41 +- .../iotdb/db/tools/PBTreeFileSketchTest.java | 5 + .../apache/iotdb/db/utils/DateTimeUtilsTest.java | 44 + .../apache/iotdb/db/utils/EnvironmentUtils.java | 25 +- .../org/apache/iotdb/db/utils/MemUtilsTest.java | 87 +- .../resources/conf/iotdb-common.properties | 28 +- .../commons/auth/authorizer/BasicAuthorizer.java | 10 + .../org/apache/iotdb/commons/auth/entity/Role.java | 14 +- .../iotdb/commons/auth/role/BasicRoleManager.java | 1 + .../commons/auth/role/LocalFileRoleAccessor.java | 11 + .../iotdb/commons/auth/user/BasicUserManager.java | 15 +- .../commons/auth/user/LocalFileUserAccessor.java | 11 + .../iotdb/commons/concurrent/ThreadName.java | 1 + .../threadpool/WrappedThreadPoolExecutor.java | 2 +- .../apache/iotdb/commons/conf/CommonConfig.java | 113 +- .../iotdb/commons/conf/CommonDescriptor.java | 37 + .../apache/iotdb/commons/conf/IoTDBConstant.java | 3 +- .../org/apache/iotdb/commons/path/AlignedPath.java | 8 + .../commons/pipe/agent/plugin/PipePluginAgent.java | 11 + .../commons/pipe/agent/task/PipeTaskAgent.java | 15 +- .../iotdb/commons/pipe/config/PipeConfig.java | 40 + .../config/constant/PipeConnectorConstant.java | 2 + .../pipe/config/constant/SystemConstant.java} | 15 +- .../connector/payload/request/PipeRequestType.java | 4 +- .../builtin/connector/iotdb/IoTDBConnector.java | 34 +- .../commons/pipe/task/meta/PipeMetaKeeper.java | 4 + .../commons/pipe/task/subtask/PipeSubtask.java | 4 - .../iotdb/commons/schema/MergeSortIterator.java | 122 ++ .../apache/iotdb/commons/service/ServiceType.java | 2 + .../iotdb/commons/service/ThriftService.java | 2 +- .../udf/builtin/BuiltinAggregationFunction.java | 8 +- .../commons/udf/service/UDFManagementService.java | 49 +- .../org/apache/iotdb/commons/utils/FileUtils.java | 76 ++ .../apache/iotdb/commons/utils/StatusUtils.java | 52 + .../org/apache/iotdb/commons/utils/TestOnly.java | 8 +- .../tsfile/read/common/block/TsBlockBuilder.java | 17 +- .../tsfile/read/common/block/TsBlockUtil.java | 55 + .../tsfile/read/filter/basic/ValueFilter.java | 4 + .../tsfile/read/filter/factory/ValueFilterApi.java | 8 + .../tsfile/read/reader/page/AlignedPageReader.java | 54 +- .../write/writer/RestorableTsFileIOWriter.java | 1 - .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 52 +- .../writer/TsFileIOWriterMemoryControlTest.java | 34 +- .../thrift-commons/src/main/thrift/common.thrift | 6 +- .../src/main/thrift/confignode.thrift | 22 +- .../src/main/thrift/datanode.thrift | 5 +- pom.xml | 12 +- 586 files changed, 25384 insertions(+), 7964 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/memory/MemoryManager.java index fc6010b9ebe,99985a11da9..4451febf6fd --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/memory/MemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/memory/MemoryManager.java @@@ -37,8 -37,9 +37,10 @@@ import java.util.Iterator import java.util.List; import java.util.NoSuchElementException; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + import static org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.MemoryManager.STATUS.ITERATE_NEW_BUFFER; + import static org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.MemoryManager.STATUS.ITERATE_UPDATE_BUFFER; import static org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.container.ICachedMNodeContainer.getBelongedContainer; import static org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.container.ICachedMNodeContainer.getCachedMNodeContainer;
