This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch benchants_branch_optimize_fe in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e2124c3f36431291f74bcbbf30a804f071b502e9 Merge: 576d995f63b 03d0feb6647 Author: Beyyes <[email protected]> AuthorDate: Fri Jun 2 17:10:50 2023 +0800 Merge branch 'master' into benchants_branch_optimize_fe .github/workflows/influxdb-protocol.yml | 67 ---- .github/workflows/site-build.yml | 4 +- antlr/pom.xml | 2 +- code-coverage/pom.xml | 2 +- confignode/pom.xml | 2 +- .../response/pipe/task/PipeTableResp.java | 4 + .../iotdb/confignode/manager/cq/CQManager.java | 4 +- .../runtime/PipeHandleMetaChangeProcedure.java | 48 ++- .../impl/schema/DeleteLogicalViewProcedure.java | 4 +- consensus/pom.xml | 2 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 3 +- distribution/pom.xml | 8 +- docs/UserGuide/Monitor-Alert/Metric-Tool.md | 14 +- docs/UserGuide/Query-Data/Last-Query.md | 26 +- docs/UserGuide/Query-Data/Order-By.md | 165 +++++++- docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 14 +- docs/zh/UserGuide/Query-Data/Last-Query.md | 22 +- docs/zh/UserGuide/Query-Data/Order-By.md | 168 +++++++- example/client-cpp-example/pom.xml | 2 +- example/ext-pipe-plugin-example/pom.xml | 4 +- example/flink/pom.xml | 2 +- example/hadoop/pom.xml | 2 +- example/influxdb-protocol-example/pom.xml | 39 -- .../org/apache/iotdb/influxdb/InfluxDBExample.java | 110 ------ example/jdbc/pom.xml | 2 +- example/kafka/pom.xml | 2 +- example/mqtt-customize/pom.xml | 2 +- example/mqtt/pom.xml | 2 +- example/pom.xml | 3 +- example/pulsar/pom.xml | 2 +- example/rabbitmq/pom.xml | 2 +- example/rest-java-example/pom.xml | 2 +- example/rocketmq/pom.xml | 2 +- example/session/pom.xml | 2 +- example/trigger/pom.xml | 2 +- example/tsfile/pom.xml | 2 +- example/udf/pom.xml | 2 +- integration-test/pom.xml | 7 +- .../iotdb/db/it/groupby/IoTDBGroupByCountIT.java | 7 + .../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java | 92 +++++ .../iotdb/db/it/view/IoTDBCreateAndShowViewIT.java | 294 ++++++++++++++ .../apache/iotdb/db/integration/IoTDBTtlIT.java | 354 ----------------- iotdb-api/external-api/pom.xml | 2 +- .../iotdb/external/api/ISeriesNumerMonitor.java | 46 --- iotdb-api/external-pipe-api/pom.xml | 2 +- iotdb-api/pipe-api/pom.xml | 2 +- .../java/org/apache/iotdb/pipe/api/access/Row.java | 38 +- .../event/dml/insertion/TabletInsertionEvent.java | 9 - iotdb-api/subscription-api/pom.xml | 2 +- iotdb-api/trigger-api/pom.xml | 2 +- iotdb-api/udf-api/pom.xml | 2 +- iotdb-client/cli/pom.xml | 2 +- iotdb-client/client-cpp/pom.xml | 2 +- .../client-py/iotdb/sqlalchemy/IoTDBSQLCompiler.py | 203 +++++++--- iotdb-client/client-py/pom.xml | 2 +- iotdb-client/client-py/setup.py | 4 +- iotdb-client/compile-tools/pom.xml | 2 +- iotdb-client/compile-tools/thrift/pom.xml | 2 +- iotdb-client/isession/pom.xml | 2 +- iotdb-client/jdbc/pom.xml | 2 +- iotdb-client/service-rpc/pom.xml | 2 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 4 +- iotdb-client/session/pom.xml | 2 +- iotdb-connector/flink-iotdb-connector/pom.xml | 2 +- iotdb-connector/flink-tsfile-connector/pom.xml | 2 +- iotdb-connector/grafana-connector/pom.xml | 2 +- .../grafana-plugin/pkg/plugin/plugin.go | 2 +- iotdb-connector/grafana-plugin/pom.xml | 2 +- iotdb-connector/hadoop/pom.xml | 2 +- iotdb-connector/hive-connector/pom.xml | 2 +- .../spark-iotdb-connector/scala_2.11/pom.xml | 2 +- .../spark-iotdb-connector/scala_2.12/pom.xml | 2 +- iotdb-connector/spark-tsfile/pom.xml | 2 +- iotdb-connector/zeppelin-interpreter/pom.xml | 2 +- iotdb-protocol/openapi/pom.xml | 2 +- iotdb-protocol/thrift-commons/pom.xml | 2 +- iotdb-protocol/thrift-confignode/pom.xml | 2 +- iotdb-protocol/thrift-influxdb/pom.xml | 2 +- iotdb-protocol/thrift-iot-consensus/pom.xml | 2 +- iotdb-protocol/thrift-mlnode/pom.xml | 2 +- iotdb-protocol/thrift/pom.xml | 2 +- library-udf/pom.xml | 2 +- metrics/dropwizard-metrics/pom.xml | 2 +- .../type/DropwizardHistogramSnapshot.java | 14 +- .../metrics/dropwizard/type/DropwizardTimer.java | 5 +- metrics/interface/pom.xml | 2 +- .../org/apache/iotdb/metrics/MetricConstant.java | 14 +- .../metrics/impl/DoNothingHistogramSnapshot.java | 14 +- .../apache/iotdb/metrics/impl/DoNothingTimer.java | 5 +- .../metrics/metricsets/cpu/CpuUsageMetrics.java | 300 ++++++++++++++ .../metrics/metricsets/jvm/JvmThreadMetrics.java | 4 +- .../metricsets/net/LinuxNetMetricManager.java | 4 +- .../reporter/prometheus/PrometheusReporter.java | 10 +- .../iotdb/metrics/type/HistogramSnapshot.java | 15 +- .../java/org/apache/iotdb/metrics/type/Timer.java | 5 +- metrics/micrometer-metrics/pom.xml | 4 +- .../micrometer/MicrometerMetricManager.java | 4 +- .../micrometer/type/MicrometerHistogram.java | 2 +- .../type/MicrometerHistogramSnapshot.java | 27 +- .../metrics/micrometer/type/MicrometerTimer.java | 11 +- .../type/MicrometerTimerHistogramSnapshot.java | 28 +- metrics/pom.xml | 2 +- mlnode/pom.xml | 2 +- node-commons/pom.xml | 2 +- .../resources/conf/iotdb-common.properties | 5 +- .../commons/concurrent/DataNodeThreadModule.java | 25 +- .../commons/concurrent/IoTDBThreadPoolFactory.java | 40 +- .../iotdb/commons/concurrent/ThreadName.java | 348 +++++++++++++++-- .../commons/concurrent/ThreadPoolMetrics.java | 176 +++++++++ .../WrappedScheduledExecutorService.java | 2 + .../WrappedSingleThreadExecutorService.java | 54 +++ .../WrappedSingleThreadExecutorServiceMBean.java | 2 +- .../WrappedSingleThreadScheduledExecutor.java | 54 +++ .../WrappedSingleThreadScheduledExecutorMBean.java | 2 +- .../threadpool/WrappedThreadPoolExecutor.java | 3 + .../apache/iotdb/commons/conf/CommonConfig.java | 9 + .../iotdb/commons/conf/CommonDescriptor.java | 14 +- .../PipeRuntimeConnectorCriticalException.java | 73 ++++ .../pipe}/PipeRuntimeCriticalException.java | 36 +- .../exception/pipe}/PipeRuntimeException.java | 11 +- .../exception/pipe/PipeRuntimeExceptionType.java | 82 ++++ .../pipe}/PipeRuntimeNonCriticalException.java | 36 +- .../iotdb/commons/pipe/config/PipeConfig.java | 4 + .../builtin/processor/DoNothingProcessor.java | 13 +- .../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 35 +- .../commons/pipe/utils/PipeBinaryTransformer.java | 24 +- .../pipe/utils/PipeDataTypeTransformer.java | 94 +++++ .../commons/schema/view/LogicalViewSchema.java | 7 +- pom.xml | 2 +- server/pom.xml | 2 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 8 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 - .../org/apache/iotdb/db/engine/StorageEngine.java | 11 +- .../iotdb/db/engine/TsFileMetricManager.java | 163 -------- .../execute/task/CrossSpaceCompactionTask.java | 43 ++- .../execute/task/InnerSpaceCompactionTask.java | 23 +- .../compaction/execute/utils/CompactionUtils.java | 16 +- .../compaction/schedule/CompactionTaskManager.java | 4 +- .../engine/flush/pool/FlushSubTaskPoolManager.java | 6 +- .../db/engine/flush/pool/FlushTaskPoolManager.java | 5 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 60 ++- .../db/engine/storagegroup/TsFileResourceList.java | 10 +- .../metadata/view/BrokenViewException.java} | 29 +- .../metadata/view/ViewNotExistException.java | 70 ++++ .../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 101 +++-- .../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 101 +++-- .../db/metadata/mtree/traverser/Traverser.java | 7 - .../mtree/traverser/basic/EntityTraverser.java | 31 +- .../traverser/basic/MeasurementTraverser.java | 36 +- .../schemaregion/result/ShowTimeSeriesResult.java | 5 + .../metadata/query/info/ITimeSeriesSchemaInfo.java | 2 + .../metadata/query/info/TimeseriesSchemaInfo.java | 92 +++++ .../reader/TimeseriesReaderWithViewFetch.java | 186 +++++++++ .../metadata/schemaregion/ISchemaRegionParams.java | 3 - .../db/metadata/schemaregion/SchemaEngine.java | 27 +- .../schemaregion/SchemaRegionMemoryImpl.java | 106 ++--- .../metadata/schemaregion/SchemaRegionParams.java | 12 +- .../schemaregion/SchemaRegionSchemaFileImpl.java | 86 ++--- .../apache/iotdb/db/metadata/utils/MetaUtils.java | 11 +- .../visitor/GetSourcePathsVisitor.java | 4 + .../db/metadata/visitor/DeviceFilterVisitor.java | 28 +- .../metadata/visitor/TimeseriesFilterVisitor.java | 46 +++ .../db/mpp/aggregation/AccumulatorFactory.java | 2 +- .../iotdb/db/mpp/execution/driver/DataDriver.java | 2 +- .../db/mpp/execution/driver/SchemaDriver.java | 2 +- .../execution/exchange/MPPDataExchangeService.java | 4 +- .../mpp/execution/exchange/SharedTsBlockQueue.java | 6 +- .../fragment/FragmentInstanceContext.java | 25 ++ .../fragment/FragmentInstanceExecution.java | 2 +- .../fragment/FragmentInstanceManager.java | 9 +- .../last/AlignedUpdateLastCacheOperator.java | 17 +- .../AlignedUpdateViewPathLastCacheOperator.java | 57 +++ .../process/last/UpdateLastCacheOperator.java | 10 +- .../last/UpdateViewPathLastCacheOperator.java | 50 +++ .../operator/schema/SchemaQueryScanOperator.java | 1 - .../operator/schema/source/DeviceSchemaSource.java | 5 - .../operator/schema/source/ISchemaSource.java | 10 - .../schema/source/LogicalViewSchemaSource.java | 111 +----- .../operator/schema/source/NodeSchemaSource.java | 5 - .../schema/source/PathsUsingTemplateSource.java | 5 - .../schema/source/TimeSeriesSchemaSource.java | 113 +----- .../db/mpp/execution/schedule/DriverScheduler.java | 17 +- .../mpp/execution/schedule/DriverTaskThread.java | 4 +- .../org/apache/iotdb/db/mpp/plan/Coordinator.java | 12 +- .../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 36 +- .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 219 ++++++----- .../db/mpp/plan/analyze/ExpressionAnalyzer.java | 72 +--- .../mpp/plan/analyze/ExpressionTypeAnalyzer.java | 23 +- .../iotdb/db/mpp/plan/analyze/ExpressionUtils.java | 194 ++++++---- .../config/executor/ClusterConfigTaskExecutor.java | 4 +- .../iotdb/db/mpp/plan/expression/Expression.java | 80 ++-- .../plan/expression/binary/BinaryExpression.java | 8 +- .../plan/expression/binary/WhenThenExpression.java | 11 +- .../db/mpp/plan/expression/leaf/LeafOperand.java | 2 +- .../plan/expression/leaf/TimeSeriesOperand.java | 2 +- .../plan/expression/multi/FunctionExpression.java | 14 +- .../expression/other/CaseWhenThenExpression.java | 15 +- .../plan/expression/ternary/BetweenExpression.java | 14 +- .../plan/expression/ternary/TernaryExpression.java | 2 +- .../db/mpp/plan/expression/unary/InExpression.java | 23 +- .../plan/expression/unary/IsNullExpression.java | 6 +- .../mpp/plan/expression/unary/LikeExpression.java | 6 +- .../plan/expression/unary/LogicNotExpression.java | 15 +- .../plan/expression/unary/NegationExpression.java | 10 +- .../plan/expression/unary/RegularExpression.java | 6 +- .../mpp/plan/expression/unary/UnaryExpression.java | 2 +- .../BindTypeForTimeSeriesOperandVisitor.java | 1 + .../visitor/ColumnTransformerVisitor.java | 4 +- .../visitor/CompleteMeasurementSchemaVisitor.java | 9 +- .../visitor/ExpressionAnalyzeVisitor.java | 2 +- .../visitor/GetMeasurementExpressionVisitor.java | 16 +- .../visitor/IntermediateLayerVisitor.java | 2 +- .../expression/visitor/ReconstructVisitor.java | 7 +- .../visitor/RemoveAliasFromExpressionVisitor.java | 19 +- .../RemoveWildcardAndViewInExpressionVisitor.java | 95 ----- .../RemoveWildcardAndViewInFilterVisitor.java | 109 ------ .../ReplaceRawPathWithGroupedPathVisitor.java | 2 +- .../BindSchemaForExpressionVisitor.java} | 64 ++- .../BindSchemaForPredicateVisitor.java} | 31 +- .../{ => cartesian}/CartesianProductVisitor.java | 9 +- ...atDeviceAndBindSchemaForExpressionVisitor.java} | 8 +- ...catDeviceAndBindSchemaForPredicateVisitor.java} | 8 +- .../ConcatExpressionWithSuffixPathsVisitor.java | 4 +- .../db/mpp/plan/planner/LogicalPlanBuilder.java | 119 ++++-- .../db/mpp/plan/planner/LogicalPlanVisitor.java | 27 +- .../db/mpp/plan/planner/OperatorTreeGenerator.java | 127 +++--- .../db/mpp/plan/planner/SubPlanTypeExtractor.java | 2 +- .../plan/planner/distribution/SourceRewriter.java | 44 +-- .../planner/plan/node/process/TransformNode.java | 2 +- .../plan/node/process/last/LastQueryMergeNode.java | 48 ++- .../plan/node/process/last/LastQueryNode.java | 56 ++- .../plan/node/source/AlignedLastQueryScanNode.java | 44 ++- .../plan/node/source/LastQueryScanNode.java | 48 ++- .../plan/parameter/AggregationDescriptor.java | 4 +- .../CrossSeriesAggregationDescriptor.java | 6 +- .../planner/plan/parameter/GroupByParameter.java | 2 + .../plan/statement/component/HavingCondition.java | 2 +- .../plan/statement/component/OrderByComponent.java | 2 +- .../db/mpp/plan/statement/component/Ordering.java | 10 + .../mpp/plan/statement/component/ResultColumn.java | 8 +- .../plan/statement/component/SelectComponent.java | 2 +- .../plan/statement/component/WhereCondition.java | 2 +- .../db/mpp/plan/statement/crud/QueryStatement.java | 12 +- .../dag/intermediate/IntermediateLayer.java | 2 +- .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 2 +- .../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 4 +- .../core/collector/IoTDBDataRegionCollector.java | 6 +- .../PipeHistoricalDataRegionTsFileCollector.java | 15 +- .../PipeRealtimeDataRegionHybridCollector.java | 21 +- .../PipeRealtimeDataRegionLogCollector.java | 12 +- .../PipeRealtimeDataRegionTsFileCollector.java | 12 +- .../realtime/assigner/PipeDataRegionAssigner.java | 2 +- .../impl/iotdb/v1/IoTDBThriftConnectorV1.java | 49 ++- .../impl/iotdb/v1/IoTDBThriftReceiverV1.java | 59 ++- .../connector/impl/iotdb/v1/PipeRequestType.java | 5 +- .../iotdb/v1/request/PipeTransferTabletReq.java | 222 +++++++++++ .../manager/PipeConnectorSubtaskLifeCycle.java | 25 +- .../manager/PipeConnectorSubtaskManager.java | 15 +- .../iotdb/db/pipe/core/event/EnrichedEvent.java | 24 +- ...vent.java => PipeInsertNodeInsertionEvent.java} | 64 ++- .../core/event/impl/PipeTabletInsertionEvent.java | 111 ++---- .../core/event/impl/PipeTsFileInsertionEvent.java | 30 +- .../event/realtime/PipeRealtimeCollectEvent.java | 31 +- .../realtime/PipeRealtimeCollectEventFactory.java | 6 +- .../core/event/realtime/TsFileEpochManager.java | 12 +- .../db/pipe/core/event/view/access/PipeRow.java | 96 +++-- .../core/event/view/access/PipeRowIterator.java | 60 --- .../event/view/collector/PipeEventCollector.java | 17 +- .../event/view/collector/PipeRowCollector.java | 45 ++- .../TabletInsertionDataContainer.java | 430 +++++++++++++++++++++ .../TsFileInsertionDataContainer.java | 136 +++++++ .../TsFileInsertionDataTabletIterator.java | 287 ++++++++++++++ .../core/processor/PipeDoNothingProcessor.java | 44 ++- .../db/pipe/resource/wal/PipeWALResource.java | 4 +- .../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 3 +- .../db/pipe/task/queue/BlockingPendingQueue.java | 73 ++++ ...Queue.java => BoundedBlockingPendingQueue.java} | 9 +- .../task/queue/ListenableBlockingPendingQueue.java | 176 --------- .../queue/PendingQueueEmptyToNotEmptyListener.java | 26 -- .../queue/PendingQueueNotFullToFullListener.java | 26 -- ...eue.java => UnboundedBlockingPendingQueue.java} | 5 +- .../db/pipe/task/stage/PipeTaskCollectorStage.java | 8 +- .../db/pipe/task/stage/PipeTaskConnectorStage.java | 10 +- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 76 +--- .../db/pipe/task/subtask/PipeConnectorSubtask.java | 25 +- .../db/pipe/task/subtask/PipeProcessorSubtask.java | 4 +- .../iotdb/db/pipe/task/subtask/PipeSubtask.java | 23 +- .../db/query/control/QueryResourceManager.java | 2 +- .../apache/iotdb/db/quotas/DataNodeSizeStore.java | 7 +- .../iotdb/db/rescon/PrimitiveArrayManager.java | 22 +- .../apache/iotdb/db/service/IoTDBShutdownHook.java | 5 + .../org/apache/iotdb/db/service/SettleService.java | 3 +- .../db/service/basic/QueryFrequencyRecorder.java | 4 +- .../db/service/metrics/DataNodeMetricsHelper.java | 26 +- .../iotdb/db/service/metrics/FileMetrics.java | 321 +++++++++++++-- .../iotdb/db/service/metrics/ProcessMetrics.java | 22 +- .../iotdb/db/service/metrics/SystemMetrics.java | 7 +- .../org/apache/iotdb/db/utils/DateTimeUtils.java | 4 +- .../apache/iotdb/db/utils/TypeInferenceUtils.java | 3 +- .../ReadPointAlignedCrossCompactionTest.java | 2 +- .../ReadPointCompactionPerformerTest.java | 2 +- .../ReadPointNonAlignedCrossCompactionTest.java | 2 +- .../cross/CrossSpaceCompactionExceptionTest.java | 2 +- .../CrossSpaceCompactionWithFastPerformerTest.java | 2 +- ...eCompactionWithFastPerformerValidationTest.java | 2 +- ...sSpaceCompactionWithReadPointPerformerTest.java | 2 +- ...actionWithReadPointPerformerValidationTest.java | 2 +- .../RewriteCrossSpaceCompactionRecoverTest.java | 2 +- ...eCrossSpaceCompactionWithFastPerformerTest.java | 2 +- ...sSpaceCompactionWithReadPointPerformerTest.java | 2 +- .../InnerUnseqCompactionWithFastPerformerTest.java | 2 +- ...rUnseqCompactionWithReadPointPerformerTest.java | 2 +- ...eCrossSpaceCompactionRecoverCompatibleTest.java | 2 +- .../utils/MultiTsFileDeviceIteratorTest.java | 2 +- .../db/metadata/idtable/entry/SchemaEntryTest.java | 2 +- .../apache/iotdb/db/metric/MetricServiceTest.java | 6 +- .../iotdb/db/mpp/execution/DataDriverTest.java | 1 + .../iotdb/db/mpp/plan/analyze/AnalyzeTest.java | 6 +- .../mpp/plan/analyze/ExpressionAnalyzerTest.java | 4 +- .../db/mpp/plan/plan/QueryLogicalPlanUtil.java | 24 +- .../mpp/plan/plan/distribution/LastQueryTest.java | 9 +- .../collector/CachedSchemaPatternMatcherTest.java | 4 +- .../core/collector/PipeRealtimeCollectTest.java | 13 +- .../executor/PipeConnectorSubtaskExecutorTest.java | 6 +- .../executor/PipeProcessorSubtaskExecutorTest.java | 2 - .../datastructure/PrimitiveArrayManagerTest.java | 75 +++- site/pom.xml | 66 +++- site/src/main/.vuepress/navbar/en.ts | 1 + site/src/main/.vuepress/navbar/zh.ts | 1 + site/src/main/.vuepress/sidebar/{ => V1.2.x}/en.ts | 16 +- site/src/main/.vuepress/sidebar/{ => V1.2.x}/zh.ts | 16 +- site/src/main/.vuepress/sidebar/en.ts | 2 + site/src/main/.vuepress/sidebar/zh.ts | 2 + testcontainer/pom.xml | 2 +- tsfile/pom.xml | 2 +- .../tsfile/file/metadata/enums/TSDataType.java | 7 +- .../apache/iotdb/tsfile/write/record/Tablet.java | 108 +++--- .../iotdb/tsfile/write/record/TabletTest.java | 14 +- 338 files changed, 6571 insertions(+), 3612 deletions(-)
