This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch load_v2 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ff3887a6a683646ebeae30f9015aa5bfe6c094ef Merge: 8805ca8f0a8 5732861f698 Author: Tian Jiang <[email protected]> AuthorDate: Mon Oct 30 09:08:02 2023 +0800 Merge branch 'master' into load_v2 # Conflicts: # iotdb-core/datanode/pom.xml # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeConnectorConstant.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/historical/PipeHistoricalDataRegionTsFileExtractor.java .github/workflows/grafana-plugin.yml | 10 +- LICENSE-binary | 1 + README.md | 2 +- README_ZH.md | 2 +- code-coverage/pom.xml | 72 +- distribution/pom.xml | 165 +- .../DockerCompose/docker-compose-cluster-1c2d.yml | 6 +- .../DockerCompose/docker-compose-host-3c3d.yml | 4 +- .../DockerCompose/docker-compose-standalone.yml | 4 +- docker/src/main/Dockerfile-1c1d | 4 +- example/client-cpp-example/pom.xml | 28 +- example/flink-sql/pom.xml | 2 +- example/flink/pom.xml | 22 +- example/hadoop/pom.xml | 2 +- example/jdbc/pom.xml | 4 +- example/pom.xml | 62 +- example/pulsar/pom.xml | 2 +- example/rest-java-example/pom.xml | 7 +- example/rocketmq/pom.xml | 2 +- example/{session => schema}/pom.xml | 14 +- .../org/apache/iotdb/schema/PathCheckExample.java | 138 ++ example/session/pom.xml | 3 +- example/trigger/pom.xml | 10 +- example/udf/pom.xml | 4 +- integration-test/checkstyle.xml | 2 +- integration-test/pom.xml | 592 +++-- .../iotdb/it/env/cluster/env/AbstractEnv.java | 18 +- .../it/env/cluster/node/AbstractNodeWrapper.java | 18 +- .../it/env/cluster/node/ConfigNodeWrapper.java | 9 +- .../iotdb/it/env/cluster/node/DataNodeWrapper.java | 4 +- .../iotdb/it/env/remote/env/RemoteServerEnv.java | 7 +- .../iotdb/it/framework/IoTDBTestConstants.java | 10 +- .../iotdb/it/framework/IoTDBTestReporter.java | 4 +- .../apache/iotdb/it/framework/IoTDBTestStat.java | 4 +- .../apache/iotdb/cli/it/StartClientScriptIT.java | 11 +- .../it/cluster/IoTDBClusterRestartIT.java | 10 +- .../db/it/IoTDBSyntaxConventionIdentifierIT.java | 14 +- .../IoTDBOrderByLimitOffsetAlignByDeviceIT.java | 102 + .../IoTDBOrderByWithAlignByDeviceIT.java | 290 ++- .../db/it/aligned/IoTDBGroupByLevelQueryIT.java | 12 +- .../db/it/aligned/IoTDBInsertAlignedValues2IT.java | 42 + .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 95 +- .../iotdb/db/it/auth/IoTDBSystemPermissionIT.java | 7 +- .../db/it/auth/IoTDBTemplatePermissionIT.java | 32 +- .../db/it/groupby/IoTDBGroupByNaturalMonthIT.java | 110 +- .../it/schema/IoTDBClusterMeasurementQuotaIT.java | 16 +- .../db/it/schema/IoTDBDeactivateTemplateIT.java | 56 +- .../db/it/schema/IoTDBDeleteTimeSeriesIT.java | 4 +- .../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 26 +- .../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 234 +- .../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 10 +- .../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 159 +- .../db/it/withoutNull/IoTDBWithoutAllNullIT.java | 4 +- .../iotdb/it/framework/IoTDBTestListener.java | 7 +- .../apache/iotdb/it/framework/IoTDBTestRunner.java | 0 .../framework/IoTDBTestRunnerWithParameters.java | 0 .../IoTDBTestRunnerWithParametersFactory.java | 0 .../apache/iotdb/libudf/it/anomaly/AnomalyIT.java | 121 +- .../iotdb/libudf/it/dprofile/DProfileIT.java | 12 +- .../iotdb/libudf/it/dquality/DQualityIT.java | 2 + .../apache/iotdb/libudf/it/drepair/DRepairIT.java | 2 + .../iotdb/libudf/it/frequency/FrequencyIT.java | 2 + .../org/apache/iotdb/tools/ExportCsvTestIT.java | 4 +- .../org/apache/iotdb/tools/ExportTsFileTestIT.java | 4 +- .../org/apache/iotdb/util/AbstractSchemaIT.java | 4 +- .../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 5 +- iotdb-api/external-api/pom.xml | 8 +- iotdb-api/pipe-api/pom.xml | 8 +- .../api/customizer/parameter/PipeParameters.java | 135 +- iotdb-api/trigger-api/pom.xml | 14 +- iotdb-api/udf-api/pom.xml | 6 +- iotdb-client/cli/pom.xml | 178 +- .../cli/src/assembly/resources/sbin/start-cli.bat | 40 +- .../cli/src/assembly/resources/sbin/start-cli.sh | 22 +- .../main/java/org/apache/iotdb/tool/ImportCsv.java | 84 +- .../iotdb/tool/integration/ExportCsvTestIT.java | 2 +- .../iotdb/tool/integration/ImportCsvTestIT.java | 2 +- iotdb-client/client-cpp/pom.xml | 61 +- iotdb-client/client-cpp/src/main/Session.cpp | 2 +- iotdb-client/client-py/README.md | 24 +- iotdb-client/client-py/iotdb/Session.py | 14 +- .../client-py/iotdb/template/TemplateNode.py | 2 +- iotdb-client/client-py/pom.xml | 45 +- iotdb-client/client-py/tests/test_template.py | 4 +- iotdb-client/isession/pom.xml | 21 +- iotdb-client/jdbc/pom.xml | 128 +- .../test/java/org/apache/iotdb/jdbc/BatchTest.java | 2 +- .../org/apache/iotdb/jdbc/IoTDBConnectionTest.java | 2 +- .../iotdb/jdbc/IoTDBDatabaseMetadataTest.java | 2 +- .../apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java | 2 +- .../iotdb/jdbc/IoTDBPreparedStatementTest.java | 2 +- .../org/apache/iotdb/jdbc/IoTDBStatementTest.java | 2 +- iotdb-client/service-rpc/pom.xml | 104 +- iotdb-client/session/pom.xml | 105 +- .../java/org/apache/iotdb/session/Session.java | 4 +- iotdb-connector/flink-iotdb-connector/pom.xml | 35 +- .../iotdb/flink/IoTDBSinkBatchInsertTest.java | 2 +- .../iotdb/flink/IoTDBSinkBatchTimerTest.java | 2 +- .../apache/iotdb/flink/IoTDBSinkInsertTest.java | 2 +- iotdb-connector/flink-sql-iotdb-connector/pom.xml | 72 +- .../flink/sql/function/IoTDBLookupFunction.java | 4 +- iotdb-connector/flink-tsfile-connector/pom.xml | 112 +- .../src/test/resources/log4j.properties | 18 +- iotdb-connector/grafana-connector/pom.xml | 138 +- .../iotdb/web/grafana/conf/MyConfiguration.java | 15 +- .../web/grafana/interceptor/LoginInterceptor.java | 14 +- iotdb-connector/grafana-plugin/backend-compile.sh | 2 +- iotdb-connector/grafana-plugin/go.mod | 10 +- .../grafana-plugin/pkg/plugin/plugin.go | 6 +- iotdb-connector/grafana-plugin/pom.xml | 6 +- iotdb-connector/grafana-plugin/yarn.lock | 133 +- iotdb-connector/hadoop/pom.xml | 105 +- iotdb-connector/hive-connector/pom.xml | 178 +- iotdb-connector/pom.xml | 15 + iotdb-connector/spark-iotdb-connector/pom.xml | 101 +- .../spark-iotdb-connector/scala_2.11/pom.xml | 5 +- .../spark-iotdb-connector/scala_2.12/pom.xml | 7 + iotdb-connector/spark-tsfile/pom.xml | 87 +- iotdb-connector/zeppelin-interpreter/pom.xml | 38 +- iotdb-core/antlr/pom.xml | 20 +- .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 2 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 50 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 13 +- iotdb-core/confignode/pom.xml | 150 +- .../src/assembly/resources/conf/confignode-env.bat | 72 +- .../src/assembly/resources/conf/confignode-env.sh | 113 +- .../resources/conf/iotdb-confignode.properties | 9 +- .../heartbeat/ConfigNodeHeartbeatHandler.java | 19 +- .../heartbeat/DataNodeHeartbeatHandler.java | 8 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 41 +- .../confignode/conf/ConfigNodeDescriptor.java | 59 +- .../confignode/conf/ConfigNodeStartupCheck.java | 10 +- .../confignode/conf/SystemPropertiesUtils.java | 14 +- .../consensus/request/ConfigPhysicalPlan.java | 17 + .../iotdb/confignode/manager/ConfigManager.java | 4 +- .../manager/consensus/ConsensusManager.java | 4 +- .../manager/node/ClusterNodeStartUtils.java | 4 +- .../iotdb/confignode/manager/node/NodeManager.java | 5 + .../iotdb/confignode/persistence/AuthorInfo.java | 6 +- .../iotdb/confignode/persistence/TriggerInfo.java | 2 + .../iotdb/confignode/persistence/UDFInfo.java | 3 + .../persistence/pipe/PipePluginInfo.java | 10 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 1 + .../persistence/schema/ClusterSchemaInfo.java | 9 +- .../persistence/schema/TemplatePreSetTable.java | 18 +- .../persistence/schema/TemplateTable.java | 19 +- .../impl/schema/DeactivateTemplateProcedure.java | 2 +- .../iotdb/confignode/service/ConfigNode.java | 14 +- .../confignode/service/ConfigNodeShutdownHook.java | 6 +- .../confignode/persistence/AuthorInfoTest.java | 987 +++----- .../confignode1conf/iotdb-confignode.properties | 2 +- .../confignode2conf/iotdb-confignode.properties | 2 +- .../confignode3conf/iotdb-confignode.properties | 2 +- iotdb-core/consensus/pom.xml | 140 +- .../apache/iotdb/consensus/config/RatisConfig.java | 53 +- .../iot/client/AsyncIoTConsensusServiceClient.java | 6 +- .../consensus/iot/client/DispatchLogHandler.java | 18 +- .../apache/iotdb/consensus/ratis/DiskGuardian.java | 252 +++ .../iotdb/consensus/ratis/RatisConsensus.java | 71 +- .../ratis/metrics/IoTDBMetricRegistry.java | 2 +- .../consensus/ratis/utils/RatisLogMonitor.java | 88 - .../iotdb/consensus/ratis/DiskGuardianTest.java | 120 + .../iotdb/consensus/ratis/RatisConsensusTest.java | 4 +- .../apache/iotdb/consensus/ratis/TestUtils.java | 7 +- iotdb-core/datanode/pom.xml | 582 +++-- .../src/assembly/resources/conf/datanode-env.bat | 69 +- .../src/assembly/resources/conf/datanode-env.sh | 105 +- .../resources/conf/iotdb-datanode.properties | 20 +- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 20 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 77 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 118 +- .../db/consensus/DataRegionConsensusImpl.java | 4 +- .../db/consensus/SchemaRegionConsensusImpl.java | 4 +- .../schemaregion/SchemaExecutionVisitor.java | 2 +- .../exception/metadata/PathNotExistException.java | 4 +- .../metadata/SeriesOverflowException.java | 5 +- .../template/TemplateIncompatibleException.java | 4 +- .../db/pipe/agent/plugin/PipePluginAgent.java | 11 +- .../db/pipe/agent/receiver/PipeReceiverAgent.java | 14 +- .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 8 +- .../config/constant/PipeConnectorConstant.java | 27 + .../config/constant/PipeExtractorConstant.java | 31 +- .../builder/PipeTransferBatchReqBuilder.java | 9 +- .../request/PipeTransferTabletRawReq.java | 1 + .../db/pipe/connector/protocol/IoTDBConnector.java | 40 +- .../protocol/airgap/IoTDBAirGapConnector.java | 8 +- .../protocol/legacy/IoTDBLegacyPipeConnector.java | 37 +- .../connector/protocol/opcua/OpcUaConnector.java | 26 +- .../protocol/opcua/OpcUaServerBuilder.java | 20 +- .../protocol/websocket/WebSocketConnector.java | 5 +- .../apache/iotdb/db/pipe/event/EnrichedEvent.java | 17 +- .../event/common/heartbeat/PipeHeartbeatEvent.java | 11 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 24 +- .../db/pipe/event/realtime/PipeRealtimeEvent.java | 9 + .../event/realtime/PipeRealtimeEventFactory.java | 4 +- .../pipe/extractor/IoTDBDataRegionExtractor.java | 131 +- .../PipeHistoricalDataRegionExtractor.java | 2 + .../PipeHistoricalDataRegionTsFileExtractor.java | 87 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 72 +- .../PipeRealtimeDataRegionHybridExtractor.java | 21 +- .../PipeRealtimeDataRegionLogExtractor.java | 45 +- .../realtime/assigner/DisruptorQueue.java | 23 +- .../realtime/assigner/PipeDataRegionAssigner.java | 24 +- .../pipe/extractor/realtime/epoch/TsFileEpoch.java | 26 +- .../realtime/epoch/TsFileEpochManager.java | 4 + .../listener/PipeInsertionDataNodeListener.java | 9 +- .../iotdb/db/pipe/metric/PipeAssignerMetrics.java | 165 ++ .../iotdb/db/pipe/metric/PipeConnectorMetrics.java | 253 +++ .../iotdb/db/pipe/metric/PipeEventCounter.java | 79 + .../iotdb/db/pipe/metric/PipeExtractorMetrics.java | 302 +++ .../db/pipe/metric/PipeHeartbeatEventMetrics.java | 125 + .../apache/iotdb/db/pipe/metric/PipeMetrics.java | 69 + .../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 253 +++ .../iotdb/db/pipe/metric/PipeResourceMetrics.java | 90 + .../pipe/metric/PipeWALInsertNodeCacheMetrics.java | 137 ++ .../legacy/IoTDBLegacyPipeReceiverAgent.java | 1 + .../receiver/thrift/IoTDBThriftReceiverV1.java | 41 +- .../db/pipe/resource/PipeResourceManager.java | 7 + .../db/pipe/resource/memory/PipeMemoryBlock.java | 35 +- .../db/pipe/resource/memory/PipeMemoryManager.java | 92 + .../resource/tsfile/PipeTsFileResourceManager.java | 4 + .../pipe/resource/wal/PipeWALResourceManager.java | 7 + .../pipe/task/connection/BlockingPendingQueue.java | 52 +- .../db/pipe/task/connection/EnrichedDeque.java | 33 +- .../pipe/task/connection/PipeEventCollector.java | 12 + .../db/pipe/task/stage/PipeTaskExtractorStage.java | 5 +- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 2 +- .../subtask/connector/PipeConnectorSubtask.java | 18 + .../connector/PipeConnectorSubtaskManager.java | 7 +- .../subtask/processor/PipeProcessorSubtask.java | 18 + .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 5 + .../protocol/rest/filter/AuthorizationFilter.java | 41 +- .../rest/v1/impl/GrafanaApiServiceImpl.java | 6 +- .../protocol/rest/v1/impl/RestApiServiceImpl.java | 9 +- .../rest/v2/impl/GrafanaApiServiceImpl.java | 6 +- .../protocol/rest/v2/impl/RestApiServiceImpl.java | 8 +- .../db/protocol/session/RestClientSession.java | 79 + .../protocol/thrift/impl/ClientRPCServiceImpl.java | 10 +- .../db/queryengine/common/MPPQueryContext.java | 13 + .../common/header/ColumnHeaderConstant.java | 2 +- .../db/queryengine/execution/driver/Driver.java | 12 + .../db/queryengine/execution/driver/IDriver.java | 4 + .../execution/exchange/MPPDataExchangeService.java | 3 +- .../fragment/FragmentInstanceManager.java | 5 + .../execution/fragment/QueryContext.java | 34 +- .../queryengine/execution/load/TsFileSplitter.java | 12 + .../execution/operator/AbstractOperator.java | 3 + .../execution/operator/process/TopKOperator.java | 362 +++ .../last/AlignedUpdateLastCacheOperator.java | 6 +- .../operator/source/AlignedSeriesScanOperator.java | 12 +- .../execution/schedule/DriverScheduler.java | 12 +- .../schedule/DriverTaskTimeoutSentinelThread.java | 18 +- .../execution/schedule/ITaskScheduler.java | 8 + .../multilevelqueue/MultilevelPriorityQueue.java | 111 +- .../execution/schedule/task/DriverTask.java | 22 +- .../db/queryengine/plan/analyze/Analysis.java | 11 + .../queryengine/plan/analyze/AnalyzeVisitor.java | 22 +- .../plan/analyze/LoadTsfileAnalyzer.java | 5 +- .../queryengine/plan/execution/QueryExecution.java | 8 + .../config/executor/ClusterConfigTaskExecutor.java | 12 +- .../db/queryengine/plan/parser/ASTVisitor.java | 22 +- .../plan/planner/LocalExecutionPlanner.java | 4 +- .../plan/planner/LogicalPlanBuilder.java | 223 +- .../plan/planner/LogicalPlanVisitor.java | 31 +- .../plan/planner/OperatorTreeGenerator.java | 40 + .../distribution/DistributionPlanContext.java | 6 +- .../planner/distribution/DistributionPlanner.java | 16 +- .../planner/distribution/ExchangeNodeAdder.java | 127 +- .../planner/distribution/NodeGroupContext.java | 8 +- .../plan/planner/distribution/SourceRewriter.java | 130 +- .../plan/planner/plan/FragmentInstance.java | 21 + .../plan/planner/plan/node/PlanGraphPrinter.java | 10 + .../plan/planner/plan/node/PlanNodeType.java | 6 +- .../plan/planner/plan/node/PlanVisitor.java | 5 + .../plan/node/load/LoadSingleTsFileNode.java | 2 +- .../plan/planner/plan/node/process/TopKNode.java | 166 ++ .../planner/plan/node/sink/ShuffleSinkNode.java | 12 +- .../plan/scheduler/AsyncPlanNodeSender.java | 83 +- .../plan/scheduler/AsyncSendPlanNodeHandler.java | 27 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 32 +- .../queryengine/plan/statement/StatementType.java | 1 + .../plan/statement/sys/AuthorStatement.java | 2 +- .../plan/statement/sys/ShowQueriesStatement.java | 3 +- .../plan/statement/sys/ShowVersionStatement.java | 7 +- .../schemaregion/utils/ResourceByPathUtils.java | 10 +- .../template/ClusterTemplateManager.java | 4 +- .../db/schemaengine/template/ITemplateManager.java | 6 +- .../java/org/apache/iotdb/db/service/DataNode.java | 21 +- .../db/service/DataNodeInternalRPCService.java | 3 +- .../apache/iotdb/db/service/MLNodeRPCService.java | 2 +- .../org/apache/iotdb/db/service/MQTTService.java | 7 +- .../db/service/metrics/DataNodeMetricsHelper.java | 4 + .../db/service/metrics/file/ModsFileMetrics.java | 9 +- .../db/service/metrics/file/TsFileMetrics.java | 4 +- .../db/storageengine/dataregion/DataRegion.java | 4 +- .../exception/CompactionRecoverException.java} | 13 +- .../execute/recover/CompactionRecoverManager.java | 20 - .../execute/recover/CompactionRecoverTask.java | 238 +- .../execute/task/AbstractCompactionTask.java | 138 +- .../execute/task/CrossSpaceCompactionTask.java | 164 +- .../execute/task/InnerSpaceCompactionTask.java | 189 +- .../execute/utils/log/CompactionLogAnalyzer.java | 121 +- .../execute/utils/log/CompactionLogger.java | 18 +- .../execute/utils/log/CompactionTaskStage.java | 30 +- .../execute/utils/log/SimpleCompactionLogger.java | 50 + .../execute/utils/log/TsFileIdentifier.java | 30 - .../dataregion/memtable/AbstractMemTable.java | 7 +- .../memtable/AlignedWritableMemChunk.java | 19 +- .../dataregion/memtable/TsFileProcessor.java | 2 +- .../metadata/DiskAlignedChunkMetadataLoader.java | 14 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 13 +- .../dataregion/tsfile/TsFileResource.java | 81 +- .../dataregion/tsfile/TsFileResourceBlockType.java | 39 +- .../wal/checkpoint/CheckpointManager.java | 13 +- .../dataregion/wal/checkpoint/MemTableInfo.java | 7 + .../storageengine/dataregion/wal/node/WALNode.java | 30 +- .../dataregion/wal/recover/WALNodeRecoverTask.java | 61 +- .../file/AbstractTsFileRecoverPerformer.java | 5 + .../wal/recover/file/TsFilePlanRedoer.java | 9 +- .../file/UnsealedTsFileRecoverPerformer.java | 63 +- .../dataregion/wal/utils/WALEntryHandler.java | 43 +- .../dataregion/wal/utils/WALEntryPosition.java | 51 +- .../dataregion/wal/utils/WALInsertNodeCache.java | 198 +- .../org/apache/iotdb/db/utils/DateTimeUtils.java | 32 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 29 +- .../db/utils/datastructure/MergeSortHeap.java | 4 + .../iotdb/db/utils/datastructure/MergeSortKey.java | 3 +- .../db/auth/role/LocalFileRoleAccessorTest.java | 90 +- .../db/auth/role/LocalFileRoleManagerTest.java | 21 +- .../db/auth/user/LocalFileUserAccessorTest.java | 106 +- .../db/auth/user/LocalFileUserManagerTest.java | 22 +- .../apache/iotdb/db/conf/IoTDBDescriptorTest.java | 40 +- .../extractor/IoTDBDataRegionExtractorTest.java | 8 +- .../db/pipe/extractor/PipeRealtimeExtractTest.java | 2 +- .../execution/operator/MergeSortOperatorTest.java | 163 +- ...SortOperatorTest.java => TopKOperatorTest.java} | 1009 ++------- .../schedule/DefaultDriverSchedulerTest.java | 28 +- .../DriverTaskTimeoutSentinelThreadTest.java | 19 +- .../queue/MultilevelPriorityQueueTest.java | 199 ++ .../queryengine/plan/plan/PipelineBuilderTest.java | 91 + .../AlignByDeviceOrderByLimitOffsetTest.java | 278 +++ .../plan/distribution/AlignedByDeviceTest.java | 23 + .../distribution/DistributionPlannerCycleTest.java | 81 + .../queryengine/plan/plan/distribution/Util2.java | 309 +++ .../storageengine/dataregion/DataRegionTest.java | 82 + .../compaction/AbstractCompactionTest.java | 3 + .../ReadPointCompactionPerformerTest.java | 8 +- .../CrossSpaceCompactionWithFastPerformerTest.java | 26 +- ...sSpaceCompactionWithReadPointPerformerTest.java | 26 +- ...eCrossSpaceCompactionRecoverCompatibleTest.java | 413 ---- .../SizeTieredCompactionRecoverCompatibleTest.java | 276 --- .../recover/SizeTieredCompactionRecoverTest.java | 588 +++-- .../dataregion/wal/node/WALEntryHandlerTest.java | 2 +- .../wal/recover/file/TsFilePlanRedoerTest.java | 35 +- .../wal/utils/WALInsertNodeCacheTest.java | 10 +- .../db/utils/datastructure/MergeSortHeapTest.java | 116 + .../datanode1conf/iotdb-datanode.properties | 2 +- .../datanode2conf/iotdb-datanode.properties | 2 +- .../datanode3conf/iotdb-datanode.properties | 2 +- .../src/test/resources/iotdb-datanode.properties | 1 + iotdb-core/metrics/dropwizard-metrics/pom.xml | 14 +- iotdb-core/metrics/interface/pom.xml | 81 +- .../iotdb/metrics/AbstractMetricService.java | 1 + iotdb-core/metrics/micrometer-metrics/pom.xml | 10 +- iotdb-core/metrics/pom.xml | 6 +- iotdb-core/mlnode/pom.xml | 74 +- iotdb-core/node-commons/pom.xml | 299 ++- .../resources/conf/iotdb-common.properties | 7 +- .../commons/auth/entity/PriPrivilegeType.java | 117 +- .../iotdb/commons/auth/entity/PrivilegeType.java | 3 +- .../org/apache/iotdb/commons/auth/entity/Role.java | 6 +- .../commons/auth/role/LocalFileRoleAccessor.java | 47 +- .../iotdb/commons/auth/user/BasicUserManager.java | 2 +- .../commons/auth/user/LocalFileUserAccessor.java | 65 +- .../apache/iotdb/commons/client/ThriftClient.java | 13 +- .../async/AsyncConfigNodeIServiceClient.java | 6 +- .../async/AsyncDataNodeInternalServiceClient.java | 6 +- .../AsyncDataNodeMPPDataExchangeServiceClient.java | 6 +- .../async/AsyncPipeDataTransferServiceClient.java | 6 +- .../apache/iotdb/commons/conf/CommonConfig.java | 21 +- .../iotdb/commons/conf/CommonDescriptor.java | 30 +- .../apache/iotdb/commons/conf/IoTDBConstant.java | 2 + .../commons/executable/ExecutableManager.java | 1 + .../iotdb/commons/partition/QueryExecutor.java | 6 + .../iotdb/commons/pipe/config/PipeConfig.java | 14 + .../iotdb/commons/service/metric/enums/Metric.java | 33 +- iotdb-core/tsfile/pom.xml | 182 +- .../iotdb/tsfile/file/header/ChunkGroupHeader.java | 3 + .../iotdb/tsfile/read/common/block/TsBlock.java | 46 + .../tsfile/read/common/block/TsBlockBuilder.java | 2 +- .../block/column/Int32ArrayColumnEncoder.java | 20 +- .../read/common/block/column/TsBlockSerde.java | 7 +- .../tsfile/read/reader/page/AlignedPageReader.java | 201 +- .../tsfile/read/reader/page/ValuePageReader.java | 131 ++ .../read/reader/series/PaginationController.java | 8 + .../tsfile/read/TsFileSequenceReaderTest.java | 11 +- .../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 2 + .../write/writer/RestorableTsFileIOWriterTest.java | 37 + iotdb-protocol/openapi/pom.xml | 100 +- iotdb-protocol/thrift-commons/pom.xml | 14 +- iotdb-protocol/thrift-confignode/pom.xml | 28 +- .../src/main/thrift/confignode.thrift | 19 +- iotdb-protocol/thrift-consensus/pom.xml | 28 +- iotdb-protocol/thrift-datanode/pom.xml | 46 +- .../thrift-datanode/src/main/thrift/client.thrift | 1 + iotdb-protocol/thrift-mlnode/pom.xml | 28 +- library-udf/pom.xml | 43 +- library-udf/src/assembly/tools/register-UDF.bat | 4 + library-udf/src/assembly/tools/register-UDF.sh | 4 +- pom.xml | 2377 +++++++++++++++----- 410 files changed, 15000 insertions(+), 7744 deletions(-) diff --cc LICENSE-binary index 82b6baeea32,82b6baeea32..5d4e0a24b55 --- a/LICENSE-binary +++ b/LICENSE-binary @@@ -290,6 -290,6 +290,7 @@@ org.slf4j:slf4j-ap me.tongfei:progressbar:0.7.3 com.bugsnag:bugsnag:3.6.1 org.slf4j:jcl-over-slf4j:1.7.25 ++net.ricecode:string-similarity:1.0.0 EPL 1.0 diff --cc iotdb-core/datanode/pom.xml index 1ebeb6e4be0,01b2dec2265..ef15b1cd35c --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@@ -29,13 -29,341 +29,346 @@@ <artifactId>iotdb-server</artifactId> <name>IoTDB: Core: Data-Node (Server)</name> <properties> - <iotdb.test.skip>false</iotdb.test.skip> <iotdb.it.skip>${iotdb.test.skip}</iotdb.it.skip> + <iotdb.test.skip>false</iotdb.test.skip> <iotdb.ut.skip>${iotdb.test.skip}</iotdb.ut.skip> - <jjwt-api.version>0.10.8</jjwt-api.version> - <oauth2-oidc-sdk.version>8.3</oauth2-oidc-sdk.version> - <tomcat-embed-core.version>10.0.27</tomcat-embed-core.version> </properties> + <dependencies> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>service-rpc</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-consensus</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>tsfile</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>external-api</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>openapi</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>node-commons</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>isession</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-antlr</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-thrift-commons</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>udf-api</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>trigger-api</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>metrics-interface</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-thrift</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-thrift-confignode</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-thrift-mlnode</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>pipe-api</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-session</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-collections4</artifactId> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </dependency> + <dependency> + <groupId>io.airlift</groupId> + <artifactId>units</artifactId> + </dependency> + <dependency> + <groupId>io.airlift</groupId> + <artifactId>stats</artifactId> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </dependency> + <dependency> + <groupId>org.apache.thrift</groupId> + <artifactId>libthrift</artifactId> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.ratis</groupId> + <artifactId>ratis-common</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>io.jsonwebtoken</groupId> + <artifactId>jjwt-api</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.milo</groupId> + <artifactId>stack-core</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + </dependency> + <dependency> + <groupId>jakarta.servlet</groupId> + <artifactId>jakarta.servlet-api</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.milo</groupId> + <artifactId>stack-server</artifactId> + </dependency> + <dependency> + <groupId>org.checkerframework</groupId> + <artifactId>checker-qual</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-codec-mqtt</artifactId> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-pool2</artifactId> + </dependency> + <dependency> + <groupId>org.openjdk.jol</groupId> + <artifactId>jol-core</artifactId> + </dependency> + <dependency> + <groupId>org.antlr</groupId> + <artifactId>antlr4-runtime</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util</artifactId> + </dependency> + <dependency> + <groupId>io.airlift</groupId> + <artifactId>airline</artifactId> + </dependency> + <dependency> + <groupId>io.airlift</groupId> + <artifactId>concurrent</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.milo</groupId> + <artifactId>sdk-server</artifactId> + </dependency> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + </dependency> + <dependency> + <groupId>jakarta.validation</groupId> + <artifactId>jakarta.validation-api</artifactId> + </dependency> + <dependency> + <groupId>jakarta.ws.rs</groupId> + <artifactId>jakarta.ws.rs-api</artifactId> + </dependency> + <dependency> + <groupId>org.reflections</groupId> + <artifactId>reflections</artifactId> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.containers</groupId> + <artifactId>jersey-container-servlet-core</artifactId> + </dependency> + <dependency> + <groupId>io.moquette</groupId> + <artifactId>moquette-broker</artifactId> + </dependency> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-buffer</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.inject</groupId> + <artifactId>jersey-hk2</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>com.github.ben-manes.caffeine</groupId> + <artifactId>caffeine</artifactId> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>com.lmax</groupId> + <artifactId>disruptor</artifactId> + </dependency> + <dependency> + <groupId>org.java-websocket</groupId> + <artifactId>Java-WebSocket</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <!-- These two dependencies are needed at runtime, so please not set test scope --> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>dropwizard-metrics</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>micrometer-metrics</artifactId> + <version>1.3.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + <!-- Possibly these need to move into the compile or provided scope --> + <dependency> + <groupId>io.jsonwebtoken</groupId> + <artifactId>jjwt-impl</artifactId> + <scope>test</scope> + </dependency> + <!-- Possibly these need to move into the compile or provided scope --> + <dependency> + <groupId>io.jsonwebtoken</groupId> + <artifactId>jjwt-jackson</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>net.minidev</groupId> + <artifactId>json-smart</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.ratis</groupId> + <artifactId>ratis-thirdparty-misc</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>com.nimbusds</groupId> + <artifactId>oauth2-oidc-sdk</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito2</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.awaitility</groupId> + <artifactId>awaitility</artifactId> + <scope>test</scope> + </dependency> ++ <dependency> ++ <groupId>net.ricecode</groupId> ++ <artifactId>string-similarity</artifactId> ++ <version>1.0.0</version> ++ </dependency> + </dependencies> <build> <plugins> <!--using `mvn test` to run UT, `mvn verify` to run ITs diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeConnectorConstant.java index f4536a707f8,93e28a0a040..0d823abe0e7 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeConnectorConstant.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeConnectorConstant.java @@@ -68,19 -87,16 +87,27 @@@ public class PipeConnectorConstant public static final int CONNECTOR_OPC_UA_TCP_BIND_PORT_DEFAULT_VALUE = 12686; public static final String CONNECTOR_OPC_UA_HTTPS_BIND_PORT_KEY = "connector.opcua.https.port"; + public static final String SINK_OPC_UA_HTTPS_BIND_PORT_KEY = "sink.opcua.https.port"; public static final int CONNECTOR_OPC_UA_HTTPS_BIND_PORT_DEFAULT_VALUE = 8443; + public static final String CONNECTOR_OPC_UA_SECURITY_DIR_KEY = "connector.opcua.security.dir"; + public static final String SINK_OPC_UA_SECURITY_DIR_KEY = "sink.opcua.security.dir"; + public static final String CONNECTOR_OPC_UA_SECURITY_DIR_DEFAULT_VALUE = + IoTDBDescriptor.getInstance().getConfDir() != null + ? IoTDBDescriptor.getInstance().getConfDir() + File.separatorChar + "opc_security" + : System.getProperty("user.home") + File.separatorChar + "iotdb_opc_security"; + + public static final String CONNECTOR_LOCAL_SPLIT_ENABLE_KEY = "connector.local-split.enable"; + public static final String CONNECTOR_EXTERNAL_CONFIG_NODES_KEY = + "connector.external.config-nodes"; + public static final String CONNECTOR_SPLIT_MAX_SIZE_KEY = "connector.split.max-size"; + public static final int CONNECTOR_SPLIT_MAX_SIZE_DEFAULT_VALUE = 64 * 1024 * 1024; + public static final String CONNECTOR_SPLIT_MAX_CONCURRENT_FILE_KEY = + "connector.split.max-concurrent-file"; + public static final int CONNECTOR_SPLIT_MAX_CONCURRENT_FILE_DEFAULT_VALUE = 16; + public static final String CONNECTOR_EXTERNAL_USER_NAME_KEY = "connector.external.user-name"; + public static final String CONNECTOR_EXTERNAL_USER_NAME_DEFAULT_VALUE = "root"; + private PipeConnectorConstant() { throw new IllegalStateException("Utility class"); } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/historical/PipeHistoricalDataRegionTsFileExtractor.java index 4227e5315c1,009a9f510c9..d9f62aa7730 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/historical/PipeHistoricalDataRegionTsFileExtractor.java @@@ -62,19 -70,22 +70,22 @@@ public class PipeHistoricalDataRegionTs private static final Map<Integer, Long> DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP = new HashMap<>(); private static final long PIPE_MIN_FLUSH_INTERVAL_IN_MS = 2000; - private PipeTaskMeta pipeTaskMeta; - private ProgressIndex startIndex; + protected PipeTaskMeta pipeTaskMeta; + protected ProgressIndex startIndex; - private int dataRegionId; + protected int dataRegionId; - private String pattern; + protected String pattern; + private boolean isDbNameCoveredByPattern = false; - private long historicalDataExtractionStartTime; // Event time - private long historicalDataExtractionEndTime; // Event time + protected long historicalDataExtractionStartTime; // Event time + protected long historicalDataExtractionEndTime; // Event time - private long historicalDataExtractionTimeLowerBound; // Arrival time + protected long historicalDataExtractionTimeLowerBound; // Arrival time + private boolean sloppyTimeRange; // true to disable time range filter after extraction + - private Queue<TsFileResource> pendingQueue; + protected Queue<TsFileResource> pendingQueue; @Override public void validate(PipeParameterValidator validator) {
