This is an automated email from the ASF dual-hosted git repository. chenyz pushed a commit to branch pbtree_concurrent in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit de26dd70d9ebef49f81b6e00945ea3216a8c95a8 Merge: db119a762a3 b57f320153b Author: Chen YZ <[email protected]> AuthorDate: Wed Dec 13 09:53:47 2023 +0800 Merge branch 'master' into pbtree_concurrent .github/workflows/grafana-plugin.yml | 2 +- .../apache/iotdb/flink/FlinkTsFileBatchSink.java | 4 +- .../apache/iotdb/flink/FlinkTsFileBatchSource.java | 4 +- .../apache/iotdb/flink/FlinkTsFileStreamSink.java | 4 +- .../iotdb/flink/FlinkTsFileStreamSource.java | 4 +- .../java/org/apache/iotdb/flink/TsFileUtils.java | 4 +- .../iotdb/hadoop/tsfile/TSFMRReadExample.java | 8 +- .../iotdb/hadoop/tsfile/TSMRWriteExample.java | 8 +- .../apache/iotdb/hadoop/tsfile/TsFileHelper.java | 6 +- .../iotdb/hadoop/tsfile/TsFileWriteToHDFS.java | 4 +- .../main/java/org/apache/iotdb/JDBCExample.java | 4 +- .../org/apache/iotdb/PrepareStatementDemo.java | 6 +- .../main/java/org/apache/iotdb/kafka/Consumer.java | 8 +- .../org/apache/iotdb/kafka/ConsumerThread.java | 4 +- .../main/java/org/apache/iotdb/kafka/Producer.java | 4 +- .../java/org/apache/iotdb/CountPointProcessor.java | 3 +- .../iotdb/opcua/IoTDBKeyStoreLoaderClient.java | 4 - .../org/apache/iotdb/pulsar/PulsarConsumer.java | 4 +- .../apache/iotdb/pulsar/PulsarConsumerThread.java | 4 +- .../apache/iotdb/rabbitmq/RabbitMQConsumer.java | 6 +- .../apache/iotdb/rabbitmq/RabbitMQProducer.java | 4 +- .../main/java/org/apache/iotdb/HttpExample.java | 20 +- .../main/java/org/apache/iotdb/HttpsExample.java | 20 +- .../src/main/java/org/apache/iotdb/SSLClient.java | 4 +- .../apache/iotdb/rocketmq/RocketMQConsumer.java | 6 +- .../apache/iotdb/rocketmq/RocketMQProducer.java | 5 +- example/schema/pom.xml | 2 +- .../org/apache/iotdb/schema/PathCheckExample.java | 26 +- .../org/apache/iotdb/DataMigrationExample.java | 14 +- .../iotdb/HybridTimeseriesSessionExample.java | 7 +- .../org/apache/iotdb/SessionConcurrentExample.java | 12 +- .../java/org/apache/iotdb/SessionPoolExample.java | 6 +- .../iotdb/SyntaxConventionRelatedExample.java | 7 +- .../iotdb/tsfile/TsFileForceAppendWrite.java | 8 +- .../java/org/apache/iotdb/tsfile/TsFileRead.java | 22 +- .../iotdb/tsfile/TsFileWriteAlignedWithTablet.java | 4 +- .../iotdb/tsfile/TsFileWriteWithTSRecord.java | 4 +- .../apache/iotdb/tsfile/TsFileWriteWithTablet.java | 4 +- .../iotdb/it/env/cluster/ClusterConstant.java | 2 +- .../org/apache/iotdb/it/utils/TsFileGenerator.java | 14 +- .../constant/BuiltinAggregationFunctionEnum.java | 6 + .../apache/iotdb/itbase/constant/TestConstant.java | 25 + .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 1 + .../iotdb/itbase/runtime/ClusterTestStatement.java | 10 +- .../it/partition/IoTDBPartitionCreationIT.java | 4 +- .../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 6 +- .../org/apache/iotdb/db/it/IoTDBMultiDeviceIT.java | 34 + .../db/it/aggregation/IoTDBAggregationIT.java | 10 +- .../aggregation/IoTDBAggregationSmallDataIT.java | 4 +- .../iotdb/db/it/aggregation/IoTDBVarianceIT.java | 733 ++++++++++++++++ .../IoTDBAlignByDeviceWithTemplateIT.java | 689 +++++++++++++++ .../IoTDBOrderByLimitOffsetAlignByDeviceIT.java | 7 + .../aligned/IoTDBAlignedOffsetLimitPushDownIT.java | 151 ++++ .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 100 ++- .../apache/iotdb/db/it/groupby/IOTDBGroupByIT.java | 196 +++++ .../apache/iotdb/db/it/metric/IoTDBMetricIT.java | 71 ++ .../org/apache/iotdb/db/it/utils/TestUtils.java | 53 ++ .../iotdb/libudf/it/dprofile/DProfileIT.java | 27 - .../iotdb/pipe/PipeEnvironmentException.java | 14 +- .../apache/iotdb/pipe/it/AbstractPipeDualIT.java | 60 ++ .../apache/iotdb/pipe/it/IoTDBPipeClusterIT.java | 254 +++--- .../pipe/it/IoTDBPipeConnectorParallelIT.java | 77 +- ...ipeDataSyncIT.java => IoTDBPipeDataSinkIT.java} | 157 ++-- .../it/{extractor => }/IoTDBPipeExtractorIT.java | 414 +++------ .../apache/iotdb/pipe/it/IoTDBPipeLifeCycleIT.java | 62 +- .../apache/iotdb/pipe/it/IoTDBPipeProcessorIT.java | 104 +++ .../apache/iotdb/pipe/it/IoTDBPipeProtocolIT.java | 61 +- .../iotdb/pipe/it/IoTDBPipeSingleEnvDemoIT.java | 16 +- .../iotdb/pipe/it/IoTDBPipeSwitchStatusIT.java | 51 +- .../apache/iotdb/pipe/it/IoTDBPipeSyntaxIT.java | 31 +- .../iotdb/session/it/IoTDBSessionSimpleIT.java | 64 +- .../java/org/apache/iotdb/pipe/api/access/Row.java | 10 +- .../parameter/PipeParameterValidator.java | 6 +- .../java/org/apache/iotdb/cli/AbstractCli.java | 2 +- .../java/org/apache/iotdb/tool/ExportTsFile.java | 8 +- .../iotdb/jdbc/AbstractIoTDBJDBCResultSet.java | 2 +- .../org/apache/iotdb/jdbc/IoTDBDataSource.java | 6 +- .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 86 +- .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 2 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 19 +- .../java/org/apache/iotdb/jdbc/StringUtils.java | 8 +- .../java/org/apache/iotdb/session/Session.java | 8 +- .../org/apache/iotdb/session/pool/SessionPool.java | 355 ++++---- .../java/org/apache/iotdb/session/SessionTest.java | 12 + .../session/pool/SessionPoolExceptionTest.java | 12 + .../flink/sql/client/IoTDBWebSocketClient.java | 20 +- .../sql/factory/IoTDBDynamicTableFactory.java | 6 + .../flink/sql/function/IoTDBCDCSourceFunction.java | 84 +- iotdb-connector/grafana-plugin/backend-compile.bat | 2 +- iotdb-connector/grafana-plugin/backend-compile.sh | 31 +- iotdb-connector/grafana-plugin/go.mod | 35 +- .../iotdb/spark/tsfile/NarrowConverter.scala | 56 +- .../apache/iotdb/spark/tsfile/WideConverter.scala | 56 +- .../resources/conf/iotdb-confignode.properties | 2 + .../async/AsyncDataNodeHeartbeatClientPool.java | 4 +- .../heartbeat/DataNodeHeartbeatHandler.java | 6 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 2 +- .../iotdb/confignode/manager/ConfigManager.java | 2 +- .../manager/load/balancer/RegionBalancer.java | 21 +- .../region/CopySetRegionGroupAllocator.java | 160 ---- .../region/GreedyCopySetRegionGroupAllocator.java | 191 +++++ .../region/GreedyRegionGroupAllocator.java | 31 +- .../load/cache/node/NodeHeartbeatSample.java | 7 +- .../manager/load/cache/node/NodeStatistics.java | 4 +- .../manager/load/service/HeartbeatService.java | 8 +- .../iotdb/confignode/manager/node/NodeManager.java | 9 + .../manager/partition/PartitionManager.java | 36 +- .../manager/partition/PartitionMetrics.java | 36 + .../manager/pipe/metric/PipeConfigNodeMetrics.java | 47 + .../manager/pipe/metric/PipeProcedureMetrics.java | 97 +++ .../manager/pipe/metric/PipeTaskInfoMetrics.java | 99 +++ .../manager/pipe/runtime/PipeHeartbeatParser.java | 4 +- .../manager/pipe/runtime/PipeMetaSyncer.java | 15 +- .../manager/pipe/task/PipeTaskCoordinator.java | 53 +- .../iotdb/confignode/persistence/AuthorInfo.java | 174 ++-- .../confignode/persistence/node/NodeInfo.java | 29 +- .../partition/DatabasePartitionTable.java | 37 + .../persistence/partition/PartitionInfo.java | 32 + .../confignode/persistence/pipe/PipeTaskInfo.java | 46 +- .../confignode/persistence/schema/ConfigMTree.java | 2 +- .../persistence/schema/mnode/IConfigMNode.java | 28 + .../schema/mnode/basic/ConfigBasicMNode.java | 50 +- .../mnode/impl/ConfigBasicInternalMNode.java | 37 +- .../impl/pipe/AbstractOperatePipeProcedureV2.java | 3 + .../procedure/impl/pipe/PipeTaskOperation.java | 24 +- .../impl/pipe/task/CreatePipeProcedureV2.java | 21 +- .../impl/pipe/task/DropPipeProcedureV2.java | 8 +- .../impl/pipe/task/StartPipeProcedureV2.java | 39 +- .../impl/pipe/task/StopPipeProcedureV2.java | 38 +- .../iotdb/confignode/service/ConfigNode.java | 3 + .../GreedyCopySetRegionGroupAllocatorTest.java | 203 +++++ .../router/priority/GreedyPriorityTest.java | 4 +- .../priority/LeaderPriorityBalancerTest.java | 4 +- .../manager/load/cache/NodeCacheTest.java | 8 +- .../confignode/persistence/AuthorInfoTest.java | 134 ++- .../confignode/persistence/PartitionInfoTest.java | 70 +- iotdb-core/consensus/pom.xml | 10 +- .../apache/iotdb/consensus/config/RatisConfig.java | 2 +- ...ion.java => RatisReadUnavailableException.java} | 11 +- .../consensus/iot/IoTConsensusServerImpl.java | 23 +- .../consensus/iot/IoTConsensusServerMetrics.java | 2 +- .../consensus/iot/client/DispatchLogHandler.java | 15 +- .../iot/logdispatcher/IndexController.java | 2 - .../consensus/iot/logdispatcher/LogDispatcher.java | 14 +- .../service/IoTConsensusRPCServiceProcessor.java | 29 +- .../ratis/ApplicationStateMachineProxy.java | 17 +- .../iotdb/consensus/ratis/RatisConsensus.java | 57 +- .../consensus/ratis/metrics/CounterProxy.java | 6 +- .../iotdb/consensus/ratis/metrics/GaugeProxy.java | 24 +- .../ratis/metrics/IoTDBMetricRegistry.java | 121 +-- .../ratis/metrics/MetricRegistryManager.java | 2 +- .../ratis/metrics/RatisMetricsManager.java | 20 +- .../iotdb/consensus/ratis/metrics/TimerProxy.java | 32 +- .../apache/iotdb/consensus/ratis/utils/Utils.java | 27 +- .../apache/iotdb/consensus/iot/ReplicateTest.java | 30 +- .../iotdb/consensus/ratis/RecoverReadTest.java | 12 +- .../apache/iotdb/consensus/ratis/TestUtils.java | 4 +- .../resources/conf/iotdb-datanode.properties | 10 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 42 + .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 104 ++- .../iotdb/db/conf/rest/IoTDBRestServiceConfig.java | 1 + .../db/conf/rest/IoTDBRestServiceDescriptor.java | 119 +-- .../db/consensus/SchemaRegionConsensusImpl.java | 1 + .../dataregion/DataExecutionVisitor.java | 21 +- .../dataregion/DataRegionStateMachine.java | 13 +- .../IoTConsensusDataRegionStateMachine.java | 7 +- .../db/pipe/agent/plugin/PipePluginAgent.java | 6 +- .../agent/plugin/PipeProcessorConstructor.java | 3 + .../agent/runtime/PipePeriodicalJobExecutor.java | 65 +- .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 22 + .../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 12 +- .../db/pipe/commit/PipeEventCommitManager.java | 110 +++ .../iotdb/db/pipe/commit/PipeEventCommitter.java | 98 +++ .../config/constant/PipeProcessorConstant.java | 12 + .../env/PipeTaskConnectorRuntimeEnvironment.java} | 10 +- .../env/PipeTaskExtractorRuntimeEnvironment.java | 9 +- .../env/PipeTaskProcessorRuntimeEnvironment.java} | 10 +- .../plugin/env/PipeTaskRuntimeEnvironment.java | 8 +- ...oTDBThriftAsyncPipeTransferBatchReqBuilder.java | 49 -- ...IoTDBThriftSyncPipeTransferBatchReqBuilder.java | 45 +- .../builder/PipeTransferBatchReqBuilder.java | 45 + .../request/PipeTransferTabletBatchReq.java | 2 +- .../request/PipeTransferTabletRawReq.java | 2 +- .../protocol/opcua/OpcUaKeyStoreLoader.java | 5 +- .../protocol/opcua/OpcUaServerBuilder.java | 4 +- .../thrift/async/IoTDBThriftAsyncConnector.java | 119 +-- .../PipeTransferTabletBatchEventHandler.java | 13 +- .../PipeTransferTabletInsertNodeEventHandler.java | 3 +- .../PipeTransferTabletInsertionEventHandler.java | 39 +- .../handler/PipeTransferTabletRawEventHandler.java | 3 +- .../PipeTransferTsFileInsertionEventHandler.java | 20 +- .../protocol/websocket/WebSocketConnector.java | 140 ++- .../websocket/WebSocketConnectorServer.java | 488 ++++++++--- .../apache/iotdb/db/pipe/event/EnrichedEvent.java | 39 +- .../event/common/heartbeat/PipeHeartbeatEvent.java | 10 +- .../iotdb/db/pipe/event/common/row/PipeRow.java | 5 + .../db/pipe/event/common/row/PipeRowCollector.java | 8 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 18 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 16 +- .../tablet/TabletInsertionDataContainer.java | 10 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 17 +- .../tsfile/TsFileInsertionDataContainer.java | 70 +- .../db/pipe/event/realtime/PipeRealtimeEvent.java | 8 +- .../execution/executor/PipeSubtaskExecutor.java | 1 + .../pipe/extractor/IoTDBDataRegionExtractor.java | 4 +- .../PipeHistoricalDataRegionTsFileExtractor.java | 62 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 5 +- .../realtime/assigner/PipeDataRegionAssigner.java | 2 +- .../listener/PipeInsertionDataNodeListener.java | 10 +- .../iotdb/db/pipe/metric/PipeConnectorMetrics.java | 22 + .../{PipeMetrics.java => PipeDataNodeMetrics.java} | 16 +- .../db/pipe/metric/PipeEventCommitMetrics.java | 141 +++ .../pipe/metric/PipeWALInsertNodeCacheMetrics.java | 24 + .../downsampling/DownSamplingProcessor.java | 213 +++++ .../downsampling/PartialPathLastTimeCache.java | 111 +++ .../legacy/IoTDBLegacyPipeReceiverAgent.java | 11 +- .../receiver/legacy/loader/DeletionLoader.java | 5 +- .../pipe/receiver/legacy/loader/TsFileLoader.java | 5 +- .../receiver/thrift/IoTDBThriftReceiverV1.java | 2 +- .../db/pipe/resource/memory/PipeMemoryBlock.java | 16 +- .../db/pipe/resource/memory/PipeMemoryManager.java | 37 + .../pipe/resource/memory/PipeMemoryWeighUtil.java | 58 ++ .../pipe/resource/tsfile/PipeTsFileResource.java | 217 +++++ .../resource/tsfile/PipeTsFileResourceManager.java | 210 ++++- .../db/pipe/resource/wal/PipeWALResource.java | 4 +- .../pipe/resource/wal/PipeWALResourceManager.java | 67 +- .../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 3 +- .../pipe/task/connection/PipeEventCollector.java | 10 +- .../db/pipe/task/stage/PipeTaskConnectorStage.java | 17 +- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 8 +- .../iotdb/db/pipe/task/stage/PipeTaskStage.java | 6 +- .../subtask/connector/PipeConnectorSubtask.java | 17 + .../connector/PipeConnectorSubtaskLifeCycle.java | 51 +- .../connector/PipeConnectorSubtaskManager.java | 32 +- .../rest/v1/handler/RequestValidationHandler.java | 12 - .../v1/handler/StatementConstructionHandler.java | 59 -- .../protocol/rest/v1/impl/RestApiServiceImpl.java | 47 - .../protocol/rest/v2/impl/RestApiServiceImpl.java | 36 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 8 +- .../impl/DataNodeInternalRPCServiceImpl.java | 8 +- .../db/queryengine/common/MPPQueryContext.java | 15 + .../common/schematree/ClusterSchemaTree.java | 35 +- .../queryengine/common/schematree/ISchemaTree.java | 8 + .../visitor/SchemaTreeVisitorFactory.java | 5 + .../execution/aggregation/Accumulator.java | 9 + .../execution/aggregation/AccumulatorFactory.java | 10 + .../execution/aggregation/AvgAccumulator.java | 13 + .../execution/aggregation/CountAccumulator.java | 9 + .../execution/aggregation/SumAccumulator.java | 9 + .../execution/aggregation/VarianceAccumulator.java | 273 ++++++ .../slidingwindow/SlidingWindowAggregator.java | 34 - .../SlidingWindowAggregatorFactory.java | 6 + .../SmoothQueueSlidingWindowAggregator.java | 4 +- .../db/queryengine/execution/driver/Driver.java | 4 - .../execution/driver/DriverContext.java | 8 - .../execution/exchange/MPPDataExchangeManager.java | 174 ++-- .../execution/exchange/sink/LocalSinkChannel.java | 39 +- .../execution/exchange/sink/ShuffleSinkHandle.java | 27 +- .../execution/exchange/sink/SinkChannel.java | 73 +- .../exchange/source/LocalSourceHandle.java | 49 +- .../execution/exchange/source/SourceHandle.java | 80 +- .../fragment/FragmentInstanceContext.java | 31 +- .../fragment/FragmentInstanceExecution.java | 3 +- .../fragment/FragmentInstanceManager.java | 2 +- .../queryengine/execution/memory/MemoryPool.java | 6 +- .../execution/operator/AbstractOperator.java | 8 +- .../execution/operator/AggregationUtil.java | 16 +- .../queryengine/execution/operator/Operator.java | 12 +- .../execution/operator/OperatorContext.java | 12 +- .../operator/process/AggregationOperator.java | 19 +- .../operator/process/ColumnInjectOperator.java | 115 +++ .../process/SlidingWindowAggregationOperator.java | 18 +- .../execution/operator/process/TopKOperator.java | 35 +- .../operator/process/last/LastQueryUtil.java | 8 +- .../operator/schema/SchemaFetchScanOperator.java | 13 +- .../source/AbstractDataSourceOperator.java | 15 + .../AbstractSeriesAggregationScanOperator.java | 38 +- .../AlignedSeriesAggregationScanOperator.java | 25 + .../operator/source/AlignedSeriesScanOperator.java | 34 +- .../operator/source/AlignedSeriesScanUtil.java | 116 ++- .../operator/source/LastCacheScanOperator.java | 5 + .../source/SeriesAggregationScanOperator.java | 25 + .../operator/source/SeriesScanOperator.java | 26 +- .../execution/operator/source/SeriesScanUtil.java | 40 +- .../operator/source/ShowQueriesOperator.java | 5 + .../schedule/DriverTaskTimeoutSentinelThread.java | 2 +- .../timer/RuleBasedTimeSliceAllocator.java | 69 -- .../iotdb/db/queryengine/plan/Coordinator.java | 1 - .../db/queryengine/plan/analyze/Analysis.java | 102 ++- .../queryengine/plan/analyze/AnalyzeVisitor.java | 174 ++-- .../plan/analyze/ExpressionAnalyzer.java | 217 ----- .../plan/analyze/ExpressionTypeAnalyzer.java | 11 + .../queryengine/plan/analyze/ExpressionUtils.java | 105 +-- .../plan/analyze/LoadTsfileAnalyzer.java | 268 +++--- .../queryengine/plan/analyze/PredicateUtils.java | 347 ++++++++ .../queryengine/plan/analyze/TemplatedAnalyze.java | 426 +++++++++ .../db/queryengine/plan/analyze/TemplatedInfo.java | 275 ++++++ .../db/queryengine/plan/analyze/TypeProvider.java | 40 +- .../analyze/cache/partition/PartitionCache.java | 12 +- .../analyze/schema/AutoCreateSchemaExecutor.java | 25 +- .../analyze/schema/ClusterSchemaFetchExecutor.java | 17 +- .../plan/analyze/schema/ClusterSchemaFetcher.java | 13 +- .../plan/analyze/schema/ISchemaFetcher.java | 6 +- .../queryengine/plan/execution/QueryExecution.java | 1 + .../config/executor/ClusterConfigTaskExecutor.java | 3 +- .../memory/StatementMemorySourceVisitor.java | 3 +- .../db/queryengine/plan/expression/Expression.java | 5 + .../plan/expression/ExpressionFactory.java | 53 ++ .../plan/expression/ExpressionType.java | 3 +- .../expression/other/GroupByTimeExpression.java | 159 ++++ .../plan/expression/unary/InExpression.java | 15 +- .../plan/expression/unary/IsNullExpression.java | 5 - .../plan/expression/unary/LikeExpression.java | 84 +- .../plan/expression/unary/LogicNotExpression.java | 5 - .../plan/expression/unary/NegationExpression.java | 5 - .../plan/expression/unary/RegularExpression.java | 31 +- .../plan/expression/unary/UnaryExpression.java | 2 - .../plan/expression/visitor/ExpressionVisitor.java | 5 + .../visitor/TransformToViewExpressionVisitor.java | 10 +- .../ConvertPredicateToTimeFilterVisitor.java | 241 ++++++ .../visitor/predicate/PredicateVisitor.java | 102 +++ .../visitor/predicate/ReversePredicateVisitor.java | 148 ++++ .../plan/optimization/LimitOffsetPushDown.java | 19 +- .../plan/optimization/PlanNodePushDown.java | 76 ++ .../base/ColumnInjectionPushDown.java} | 14 +- .../db/queryengine/plan/parser/ASTVisitor.java | 12 +- .../plan/planner/LocalExecutionPlanContext.java | 26 +- .../plan/planner/LogicalPlanBuilder.java | 109 ++- .../plan/planner/LogicalPlanVisitor.java | 27 +- .../plan/planner/OperatorTreeGenerator.java | 319 +++---- .../plan/planner/SubPlanTypeExtractor.java | 13 +- .../plan/planner/TemplatedLogicalPlan.java | 206 +++++ .../plan/planner/TemplatedLogicalPlanBuilder.java | 142 +++ .../distribution/DistributionPlanContext.java | 5 + .../planner/distribution/DistributionPlanner.java | 36 +- .../planner/distribution/ExchangeNodeAdder.java | 44 +- .../SimpleFragmentParallelPlanner.java | 6 +- .../plan/planner/distribution/SourceRewriter.java | 46 +- .../distribution/WriteFragmentParallelPlanner.java | 6 +- .../plan/planner/plan/FragmentInstance.java | 41 +- .../plan/planner/plan/PlanFragment.java | 36 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 22 +- .../plan/planner/plan/node/PlanNode.java | 18 +- .../plan/planner/plan/node/PlanNodeType.java | 23 +- .../plan/planner/plan/node/PlanVisitor.java | 5 + .../plan/node/load/LoadSingleTsFileNode.java | 4 +- .../plan/node/load/LoadTsFilePieceNode.java | 8 +- .../node/metedata/read/SchemaFetchScanNode.java | 17 +- .../planner/plan/node/process/AggregationNode.java | 8 +- .../plan/node/process/ColumnInjectNode.java | 123 +++ .../plan/node/process/SingleDeviceViewNode.java | 28 + .../node/process/SlidingWindowAggregationNode.java | 56 +- .../plan/node/process/last/LastQueryNode.java | 45 +- .../plan/node/source/AlignedLastQueryScanNode.java | 8 +- .../source/AlignedSeriesAggregationScanNode.java | 85 +- .../plan/node/source/AlignedSeriesScanNode.java | 183 ++-- .../plan/node/source/LastQueryScanNode.java | 8 +- .../plan/node/source/LastSeriesSourceNode.java | 7 + .../node/source/SeriesAggregationScanNode.java | 85 +- .../node/source/SeriesAggregationSourceNode.java | 60 +- .../planner/plan/node/source/SeriesScanNode.java | 146 ++-- .../planner/plan/node/source/SeriesSourceNode.java | 4 +- .../planner/plan/node/write/DeleteDataNode.java | 4 +- .../plan/parameter/AggregationDescriptor.java | 18 + .../planner/plan/parameter/SeriesScanOptions.java | 66 +- .../scheduler/load/LoadTsFileDispatcherImpl.java | 18 +- .../crud/InsertMultiTabletsStatement.java | 9 + .../InternalBatchActivateTemplateStatement.java | 33 +- .../statement/internal/SchemaFetchStatement.java | 11 +- .../metadata/CreateContinuousQueryStatement.java | 4 +- .../template/AlterSchemaTemplateStatement.java | 4 +- .../binary/CompareBinaryColumnTransformer.java | 33 +- .../column/ternary/BetweenColumnTransformer.java | 21 +- .../binary/CompareEqualToTransformer.java | 5 +- .../binary/CompareGreaterEqualTransformer.java | 5 +- .../binary/CompareGreaterThanTransformer.java | 5 +- .../binary/CompareLessEqualTransformer.java | 5 +- .../binary/CompareLessThanTransformer.java | 5 +- .../binary/CompareNonEqualTransformer.java | 5 +- .../transformer/ternary/BetweenTransformer.java | 15 +- .../transformation/dag/util/TransformUtils.java | 41 +- .../apache/iotdb/db/schemaengine/SchemaEngine.java | 39 +- .../schemaengine/schemaregion/ISchemaRegion.java | 5 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 9 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 9 +- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 42 +- .../schemaregion/mtree/impl/mem/MemMTreeStore.java | 23 +- .../mtree/impl/mem/mnode/basic/BasicMNode.java | 26 +- .../impl/mem/mnode/factory/MemMNodeFactory.java | 11 +- .../impl/mem/mnode/impl/BasicInternalMNode.java | 83 +- .../impl/mem/mnode/impl/DatabaseDeviceMNode.java | 37 - .../mtree/impl/mem/mnode/impl/DatabaseMNode.java | 17 +- .../mtree/impl/mem/mnode/info/LogicalViewInfo.java | 2 +- .../impl/mem/snapshot/MemMTreeSnapshotUtil.java | 75 +- .../mtree/impl/pbtree/CachedMTreeStore.java | 59 +- .../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 42 +- .../impl/pbtree/mnode/basic/CachedBasicMNode.java | 25 +- .../pbtree/mnode/factory/CacheMNodeFactory.java | 11 +- .../mnode/impl/CachedBasicInternalMNode.java | 91 +- .../mnode/impl/CachedDatabaseDeviceMNode.java | 49 -- .../pbtree/mnode/impl/CachedDatabaseMNode.java | 18 +- .../impl/pbtree/mnode/impl/CachedDeviceMNode.java | 48 -- .../mtree/impl/pbtree/schemafile/InternalPage.java | 18 +- .../mtree/impl/pbtree/schemafile/SchemaPage.java | 2 +- .../mtree/impl/pbtree/schemafile/Segment.java | 478 ----------- .../impl/pbtree/schemafile/WrappedSegment.java | 708 ++++++++++++--- .../schemaregion/mtree/traverser/Traverser.java | 2 +- .../reader/impl/TimeseriesReaderWithViewFetch.java | 3 +- .../schemaregion/utils/MNodeUtils.java | 54 +- .../view/visitor/TransformToExpressionVisitor.java | 10 +- .../db/service/metrics/DataNodeMetricsHelper.java | 4 +- .../db/service/metrics/file/TsFileMetrics.java | 8 +- .../iotdb/db/storageengine/StorageEngine.java | 76 +- .../iotdb/db/storageengine/buffer/ChunkCache.java | 10 +- .../db/storageengine/dataregion/DataRegion.java | 257 +++++- .../CompactionLastTimeCheckFailedException.java} | 18 +- .../impl/ReadChunkCompactionPerformer.java | 5 +- .../execute/recover/CompactionRecoverTask.java | 27 +- .../execute/task/AbstractCompactionTask.java | 2 +- .../execute/task/InnerSpaceCompactionTask.java | 8 +- .../execute/utils/CompactionPathUtils.java} | 24 +- .../execute/utils/MultiTsFileDeviceIterator.java | 7 +- .../fast/AlignedSeriesCompactionExecutor.java | 21 +- .../fast/NonAlignedSeriesCompactionExecutor.java | 12 +- .../readchunk/AlignedSeriesCompactionExecutor.java | 12 + .../readchunk/SingleSeriesCompactionExecutor.java | 13 + .../utils/writer/AbstractCompactionWriter.java | 11 + .../writer/AbstractCrossCompactionWriter.java | 11 +- .../writer/AbstractInnerCompactionWriter.java | 2 + .../utils/writer/FastInnerCompactionWriter.java | 4 + .../compaction/schedule/CompactionScheduler.java | 13 + .../compaction/schedule/CompactionWorker.java | 22 +- .../estimator/AbstractCompactionEstimator.java | 27 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 6 +- .../impl/SizeTieredCompactionSelector.java | 11 +- .../compaction/settle/SettleRequestHandler.java | 24 +- .../read/reader/chunk/DiskAlignedChunkLoader.java | 6 +- .../read/reader/chunk/DiskChunkLoader.java | 6 +- .../read/reader/chunk/MemAlignedPageReader.java | 82 +- .../read/reader/chunk/MemPageReader.java | 36 +- .../dataregion/wal/buffer/WALBuffer.java | 52 +- .../dataregion/wal/buffer/WALEntry.java | 3 + .../dataregion/wal/buffer/WALEntryType.java | 2 + .../dataregion/wal/buffer/WALInfoEntry.java | 2 + .../dataregion/wal/buffer/WALSignalEntry.java | 13 +- .../dataregion/wal/checkpoint/Checkpoint.java | 10 +- .../wal/checkpoint/CheckpointManager.java | 25 +- .../dataregion/wal/checkpoint/MemTableInfo.java | 10 +- .../storageengine/dataregion/wal/node/WALNode.java | 20 +- .../dataregion/wal/utils/WALInsertNodeCache.java | 18 +- .../rescon/disk/strategy/DirectoryStrategy.java | 4 +- .../MinFolderOccupiedSpaceFirstStrategy.java | 2 +- .../apache/iotdb/db/tools/TsFileSelfCheckTool.java | 8 +- .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 14 +- .../org/apache/iotdb/db/tools/TsFileSplitTool.java | 4 +- .../trigger/service/TriggerInformationUpdater.java | 2 +- .../java/org/apache/iotdb/db/utils/MemUtils.java | 4 +- .../org/apache/iotdb/db/utils/OpenFileNumUtil.java | 8 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 24 + .../apache/iotdb/db/utils/TimeValuePairUtils.java | 62 -- .../apache/iotdb/db/utils/TypeInferenceUtils.java | 20 +- .../db/utils/columngenerator/ColumnGenerator.java} | 11 +- .../columngenerator/ColumnGeneratorType.java} | 15 +- .../SlidingTimeColumnGenerator.java | 52 ++ .../parameter/ColumnGeneratorParameter.java | 82 ++ .../SlidingTimeColumnGeneratorParameter.java | 80 ++ .../iotdb/db/utils/constant/SqlConstant.java | 6 + .../db/auth/role/LocalFileRoleAccessorTest.java | 55 +- .../db/auth/role/LocalFileRoleManagerTest.java | 124 ++- .../db/auth/user/LocalFileUserAccessorTest.java | 74 +- .../db/auth/user/LocalFileUserManagerTest.java | 48 -- .../dataregion/DataRegionStateMachineTest.java | 78 ++ .../apache/iotdb/db/metadata/mnode/MNodeTest.java | 31 +- .../metadata/mtree/schemafile/SchemaFileTest.java | 2 +- .../schemaRegion/SchemaRegionBasicTest.java | 46 +- .../schemaRegion/SchemaRegionTemplateTest.java | 37 +- .../schemaRegion/SchemaStatisticsTest.java | 2 + .../event/TsFileInsertionDataContainerTest.java | 2 +- .../resource/PipeTsFileResourceManagerTest.java | 18 +- .../execution/aggregation/AccumulatorTest.java | 332 +++++++ .../operator/AggregationOperatorTest.java | 1 + .../AlignedSeriesAggregationScanOperatorTest.java | 14 +- .../operator/AlignedSeriesScanOperatorTest.java | 15 +- .../execution/operator/FillOperatorTest.java | 15 + .../operator/LastQueryMergeOperatorTest.java | 20 + .../execution/operator/LinearFillOperatorTest.java | 35 + .../execution/operator/OperatorMemoryTest.java | 13 +- .../operator/RawDataAggregationOperatorTest.java | 5 +- .../SeriesAggregationScanOperatorTest.java | 14 +- .../SlidingWindowAggregationOperatorTest.java | 1 + .../execution/operator/TopKOperatorTest.java | 297 +++++-- .../operator/UpdateLastCacheOperatorTest.java | 6 +- .../schema/SchemaFetchScanOperatorTest.java | 4 +- .../ConvertPredicateToTimeFilterTest.java | 148 ++++ .../predicate/PredicateRemoveNotTest.java | 95 ++ .../CaseWhenThenExpressionTest.java | 2 +- .../db/queryengine/plan/analyze/AnalyzeTest.java | 26 +- .../plan/analyze/ExpressionAnalyzerTest.java | 2 +- .../plan/analyze/FakeSchemaFetcherImpl.java | 12 +- .../plan/analyze/QueryTimePartitionTest.java | 168 ++-- .../plan/optimization/LimitOffsetPushDownTest.java | 10 +- .../plan/optimization/TestPlanBuilder.java | 8 +- .../plan/plan/FragmentInstanceSerdeTest.java | 4 +- .../plan/plan/QueryLogicalPlanUtil.java | 49 +- .../distribution/AggregationDistributionTest.java | 3 +- .../AlignByDeviceOrderByLimitOffsetTest.java | 4 +- .../distribution/DistributionPlannerCycleTest.java | 9 +- .../plan/plan/distribution/LastQueryTest.java | 2 +- .../queryengine/plan/plan/distribution/Util.java | 8 +- .../queryengine/plan/plan/distribution/Util2.java | 22 +- .../plan/plan/node/PlanGraphPrinterTest.java | 2 +- .../metadata/read/SchemaFetchMergeNodeTest.java | 1 + .../metadata/read/SchemaFetchScanNodeTest.java | 2 + .../node/process/AggregationNodeSerdeTest.java | 8 +- .../node/process/GroupByLevelNodeSerdeTest.java | 11 +- .../plan/node/process/GroupByTagNodeSerdeTest.java | 9 +- .../plan/plan/node/process/LimitNodeSerdeTest.java | 2 - .../plan/node/process/OffsetNodeSerdeTest.java | 2 - .../plan/plan/node/process/SortNodeSerdeTest.java | 2 - .../plan/node/process/TimeJoinNodeSerdeTest.java | 4 - .../source/SeriesAggregationScanNodeSerdeTest.java | 9 +- .../plan/node/source/SeriesScanNodeSerdeTest.java | 2 - .../db/storageengine/buffer/ChunkCacheTest.java | 6 +- .../storageengine/dataregion/DataRegionTest.java | 86 ++ .../compaction/CompactionOverlapCheckTest.java | 269 ++++++ .../compaction/CompactionSchedulerTest.java | 25 +- .../compaction/CompactionTaskManagerTest.java | 10 +- .../ReadPointCompactionPerformerTest.java | 2 +- .../inner/InnerSpaceCompactionSelectorTest.java | 63 +- .../utils/CompactionUpdateFileCountTest.java | 2 +- .../reader/chunk/MemAlignedChunkLoaderTest.java | 9 +- .../AlignedSeriesScanLimitOffsetPushDownTest.java | 4 +- .../series/SeriesScanLimitOffsetPushDownTest.java | 4 +- .../wal/checkpoint/CheckpointManagerTest.java | 7 +- .../dataregion/wal/node/WALEntryHandlerTest.java | 2 + .../dataregion/wal/node/WALNodeTest.java | 4 + .../wal/recover/WALRecoverManagerTest.java | 40 +- .../wal/utils/WALInsertNodeCacheTest.java | 9 +- .../metricsets/disk/LinuxDiskMetricsManager.java | 12 +- .../metricsets/net/LinuxNetMetricManager.java | 12 +- iotdb-core/node-commons/pom.xml | 5 + .../resources/conf/iotdb-common.properties | 10 +- .../src/assembly/resources/sbin/iotdb-common.sh | 15 +- .../commons/auth/authorizer/BasicAuthorizer.java | 10 +- .../commons/auth/entity/PriPrivilegeType.java | 52 +- .../iotdb/commons/auth/role/BasicRoleManager.java | 92 +- .../iotdb/commons/auth/user/BasicUserManager.java | 86 +- .../apache/iotdb/commons/client/ClientManager.java | 7 +- .../iotdb/commons/concurrent/ThreadName.java | 6 +- .../apache/iotdb/commons/conf/CommonConfig.java | 2 +- .../apache/iotdb/commons/path/PathPatternTree.java | 8 + .../iotdb/commons/path/fa/dfa/PatternDFA.java | 18 +- .../iotdb/commons/path/fa/dfa/graph/DFAGraph.java | 75 +- .../ConcurrentIterableLinkedQueue.java | 345 ++++++++ .../pipe/plugin/builtin/BuiltinPipePlugin.java | 47 +- .../builtin/processor/DownSamplingProcessor.java} | 17 +- .../builtin/processor/PlaceHolderProcessor.java | 68 ++ .../commons/pipe/task/meta/PipeMetaKeeper.java | 32 + .../apache/iotdb/commons/schema/node/IMNode.java | 12 +- .../node/common/AbstractAboveDatabaseMNode.java | 18 +- .../node/common/AbstractDatabaseDeviceMNode.java | 299 ------- .../schema/node/common/AbstractDatabaseMNode.java | 26 +- .../node/common/AbstractMeasurementMNode.java | 19 +- ...actDeviceMNode.java => DeviceMNodeWrapper.java} | 130 ++- .../commons/schema/node/role/IDeviceMNode.java | 2 +- .../commons/schema/node/role/IInternalMNode.java} | 37 +- .../commons/schema/node/visitor/MNodeVisitor.java | 7 - .../viewExpression/unary/LikeViewExpression.java | 20 +- .../unary/RegularViewExpression.java | 24 +- .../iotdb/commons/service/metric/enums/Metric.java | 7 + .../udf/builtin/BuiltinAggregationFunction.java | 20 +- .../org/apache/iotdb/commons/utils/AuthUtils.java | 48 +- .../org/apache/iotdb/commons/utils/FileUtils.java | 16 +- .../org/apache/iotdb/commons/utils/IOUtils.java | 28 +- .../apache/iotdb/commons/utils/SerializeUtils.java | 8 - .../apache/iotdb/commons/path/PatternDFATest.java | 10 +- .../ConcurrentIterableLinkedQueueTest.java | 439 ++++++++++ .../pipe/{ => task/meta}/PipeMetaDeSerTest.java | 7 +- ...ReaderTimeseriesMetadataIteratorException.java} | 24 +- .../iotdb/tsfile/file/header/ChunkGroupHeader.java | 2 +- .../iotdb/tsfile/file/header/PageHeader.java | 23 +- .../tsfile/file/metadata/AlignedChunkMetadata.java | 42 +- .../file/metadata/AlignedTimeSeriesMetadata.java | 57 +- .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 22 + .../iotdb/tsfile/file/metadata/IChunkMetadata.java | 6 +- .../metadata/IMetadata.java} | 15 +- .../tsfile/file/metadata/ITimeSeriesMetadata.java | 9 +- .../tsfile/file/metadata/TimeseriesMetadata.java | 31 + .../file/metadata/enums/MetadataIndexNodeType.java | 2 +- .../file/metadata/statistics/BinaryStatistics.java | 6 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 4 + ...leSequenceReaderTimeseriesMetadataIterator.java | 220 +++++ .../apache/iotdb/tsfile/read/common/TimeRange.java | 10 +- .../iotdb/tsfile/read/common/block/TsBlock.java | 32 +- .../tsfile/read/common/block/TsBlockBuilder.java | 7 +- .../read/expression/impl/GlobalTimeExpression.java | 12 +- .../expression/impl/SingleSeriesExpression.java | 11 +- .../read/filter/PredicateRemoveNotRewriter.java | 30 +- .../iotdb/tsfile/read/filter/TimeFilter.java | 321 ------- .../iotdb/tsfile/read/filter/ValueFilter.java | 164 ---- .../tsfile/read/filter/basic/BinaryFilter.java | 95 -- .../read/filter/basic/BinaryLogicalFilter.java} | 47 +- .../DisableStatisticsTimeFilter.java} | 18 +- .../basic/DisableStatisticsValueFilter.java} | 37 +- .../iotdb/tsfile/read/filter/basic/Filter.java | 187 +++- .../tsfile/read/filter/basic/OperatorType.java | 75 ++ .../iotdb/tsfile/read/filter/basic/TimeFilter.java | 58 ++ .../tsfile/read/filter/basic/UnaryFilter.java | 97 --- .../tsfile/read/filter/basic/ValueFilter.java | 120 +++ .../tsfile/read/filter/factory/FilterFactory.java | 93 +- .../tsfile/read/filter/factory/TimeFilterApi.java | 101 +++ .../tsfile/read/filter/factory/ValueFilterApi.java | 186 ++++ .../filter/operator/{AndFilter.java => And.java} | 65 +- .../iotdb/tsfile/read/filter/operator/Between.java | 168 ---- .../iotdb/tsfile/read/filter/operator/Eq.java | 115 --- .../read/filter/{ => operator}/GroupByFilter.java | 118 +-- .../filter/{ => operator}/GroupByMonthFilter.java | 172 ++-- .../iotdb/tsfile/read/filter/operator/Gt.java | 113 --- .../iotdb/tsfile/read/filter/operator/GtEq.java | 113 --- .../iotdb/tsfile/read/filter/operator/In.java | 153 ---- .../iotdb/tsfile/read/filter/operator/Like.java | 201 ----- .../iotdb/tsfile/read/filter/operator/Lt.java | 113 --- .../iotdb/tsfile/read/filter/operator/LtEq.java | 113 --- .../filter/operator/{NotFilter.java => Not.java} | 126 ++- .../iotdb/tsfile/read/filter/operator/NotEq.java | 115 --- .../iotdb/tsfile/read/filter/operator/Or.java | 174 ++++ .../tsfile/read/filter/operator/OrFilter.java | 153 ---- .../iotdb/tsfile/read/filter/operator/Regexp.java | 199 ----- .../read/filter/operator/TimeFilterOperators.java | 601 +++++++++++++ .../read/filter/operator/ValueFilterOperators.java | 954 +++++++++++++++++++++ .../iotdb/tsfile/read/reader/IPageReader.java | 7 +- .../read/reader/chunk/AlignedChunkReader.java | 34 +- .../tsfile/read/reader/chunk/ChunkReader.java | 14 +- .../tsfile/read/reader/page/AlignedPageReader.java | 202 +++-- .../iotdb/tsfile/read/reader/page/PageReader.java | 58 +- .../tsfile/read/reader/page/ValuePageReader.java | 131 ++- .../reader/series/AbstractFileSeriesReader.java | 16 +- .../read/reader/series/EmptyFileSeriesReader.java | 4 +- .../read/reader/series/FileSeriesReader.java | 4 +- .../org/apache/iotdb/tsfile/utils/BytesUtils.java | 10 + .../FilterType.java => utils/Preconditions.java} | 24 +- .../iotdb/tsfile/utils/ReadWriteIOUtils.java | 55 +- .../org/apache/iotdb/tsfile/utils/RegexUtils.java | 92 ++ .../iotdb/tsfile/common/block/TsBlockTest.java | 77 ++ .../apache/iotdb/tsfile/read/ExpressionTest.java | 36 +- .../iotdb/tsfile/read/ReadInPartitionTest.java | 8 +- .../org/apache/iotdb/tsfile/read/ReadTest.java | 60 +- .../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 64 +- .../apache/iotdb/tsfile/read/TsFileReaderTest.java | 66 +- ...quenceReaderTimeseriesMetadataIteratorTest.java | 62 ++ .../tsfile/read/filter/FilterSerializeTest.java | 92 +- .../iotdb/tsfile/read/filter/FilterTestUtil.java | 105 +++ .../tsfile/read/filter/GroupByFilterTest.java | 33 +- .../tsfile/read/filter/GroupByMonthFilterTest.java | 88 +- .../read/filter/IExpressionOptimizerTest.java | 81 +- .../read/filter/MinTimeMaxTimeFilterTest.java | 75 +- .../iotdb/tsfile/read/filter/OperatorTest.java | 93 +- .../filter/PredicateRemoveNotRewriterTest.java | 109 +-- .../tsfile/read/filter/StatisticsFilterTest.java | 300 ++++--- .../read/query/executor/QueryExecutorTest.java | 11 +- .../read/query/timegenerator/ReadWriteTest.java | 10 +- .../timegenerator/TimeGeneratorReadEmptyTest.java | 10 +- .../timegenerator/TimeGeneratorReadWriteTest.java | 10 +- .../query/timegenerator/TimeGeneratorTest.java | 10 +- .../iotdb/tsfile/read/reader/ReaderTest.java | 8 +- iotdb-doap.rdf | 33 + .../openapi/src/main/openapi3/iotdb_rest_v1.yaml | 52 -- .../thrift-commons/src/main/thrift/common.thrift | 8 +- .../src/main/thrift/confignode.thrift | 5 + .../src/main/thrift/datanode.thrift | 8 +- library-udf/src/assembly/tools/register-UDF.bat | 1 - library-udf/src/assembly/tools/register-UDF.sh | 1 - .../apache/iotdb/library/dprofile/UDAFStddev.java | 66 -- pom.xml | 10 +- 674 files changed, 21888 insertions(+), 11396 deletions(-)
