This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch benchants_branch in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f3bfda6fd75bb3c26c043370ccc803cea155de32 Merge: d6d2e5884c5 d8324726279 Author: Beyyes <[email protected]> AuthorDate: Wed May 31 11:05:42 2023 +0800 Merge branch 'master' into benchants_branch .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 10 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 8 +- .../confignode/conf/SystemPropertiesUtils.java | 43 ++- .../partition/GreedyPartitionAllocator.java | 36 +-- .../manager/load/service/HeartbeatService.java | 8 +- .../manager/partition/PartitionManager.java | 30 +- .../manager/pipe/runtime/PipeMetaSyncer.java | 8 +- .../pipe/runtime/PipeRuntimeCoordinator.java | 5 +- .../partition/DatabasePartitionTable.java | 4 +- .../persistence/partition/PartitionInfo.java | 15 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 9 +- .../runtime/PipeHandleMetaChangeProcedure.java | 10 +- .../impl/pipe/task/CreatePipeProcedureV2.java | 7 +- .../iotdb/confignode/service/ConfigNode.java | 2 + .../request/ConfigPhysicalPlanSerDeTest.java | 9 +- .../iotdb/confignode/persistence/PipeInfoTest.java | 3 +- .../runtime/PipeHandleMetaChangeProcedureTest.java | 7 +- .../request/BatchIndexedConsensusRequest.java | 6 +- .../consensus/iot/IoTConsensusServerImpl.java | 15 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 4 +- docker/src/main/Dockerfile-1c1d | 3 + docs/UserGuide/Monitor-Alert/Metric-Tool.md | 3 +- docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 13 +- .../iotdb/hadoop/tsfile/TsFileWriteToHDFS.java | 2 +- integration-test/pom.xml | 4 + .../partition/IoTDBPartitionInheritPolicyIT.java | 26 +- .../confignode/it/utils/ConfigNodeTestUtils.java | 2 +- .../db/it/aligned/IoTDBInsertAlignedValuesIT.java | 31 ++ .../apache/iotdb/db/integration/IoTDBTtlIT.java | 354 +++++++++++++++++++++ .../api/customizer/PipeParameterValidator.java | 27 ++ .../apache/iotdb/isession/pool/ISessionPool.java | 35 ++ .../org/apache/iotdb/session/pool/SessionPool.java | 168 ++++++++++ .../util/TSFileConfigUtilCompletenessTest.java | 8 +- .../iotdb/hadoop/fileSystem/HDFSConfUtil.java | 11 +- .../apache/iotdb/hadoop/fileSystem/HDFSFile.java | 18 ++ .../apache/iotdb/hadoop/fileSystem/HDFSInput.java | 32 -- .../apache/iotdb/hadoop/tsfile/TSFHadoopTest.java | 6 +- .../apache/iotdb/hive/TSFHiveInputFormatTest.java | 4 +- .../apache/iotdb/hive/TSFHiveRecordReaderTest.java | 4 +- .../src/main/thrift/iotconsensus.thrift | 2 +- .../apache/iotdb/metrics/config/MetricConfig.java | 7 + .../iotdb/metrics/metricsets/UpTimeMetrics.java | 28 +- .../resources/conf/iotdb-common.properties | 75 +++-- .../apache/iotdb/commons/conf/CommonConfig.java | 179 ++++++++++- .../iotdb/commons/conf/CommonDescriptor.java | 109 ++++++- .../apache/iotdb/commons/conf/IoTDBConstant.java | 4 + .../index/ComparableConsensusRequest.java | 11 +- .../commons/consensus/index/ProgressIndex.java | 124 ++++++++ .../commons/consensus/index/ProgressIndexType.java | 97 ++++++ .../consensus/index/impl/HybridProgressIndex.java | 221 +++++++++++++ .../consensus/index/impl/IoTProgressIndex.java | 205 ++++++++++++ .../consensus/index/impl/MinimumProgressIndex.java | 92 ++++++ .../consensus/index/impl/RecoverProgressIndex.java | 207 ++++++++++++ .../consensus/index/impl/SimpleProgressIndex.java | 197 ++++++++++++ .../commons/partition/DataPartitionTable.java | 7 +- .../commons/partition/SeriesPartitionTable.java | 24 +- .../iotdb/commons/pipe/config/PipeConfig.java | 151 +++++++++ .../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 67 ++-- .../commons/schema/view/LogicalViewSchema.java | 32 +- .../commons/pipe/task/meta/PipeMetaDeSerTest.java | 7 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 145 +++++---- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 84 +++-- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 4 +- .../db/conf/directories/DirectoryChecker.java | 5 + .../db/conf/directories/DirectoryManager.java | 199 ------------ .../iotdb/db/conf/directories/FolderManager.java | 4 +- .../iotdb/db/conf/directories/TierManager.java | 313 ++++++++++++++++++ .../IoTConsensusDataRegionStateMachine.java | 10 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 2 +- .../performer/impl/FastCompactionPerformer.java | 7 +- .../execute/recover/CompactionRecoverManager.java | 6 +- .../execute/recover/CompactionRecoverTask.java | 2 +- .../execute/task/AbstractCompactionTask.java | 25 +- .../execute/task/CrossSpaceCompactionTask.java | 17 +- .../execute/task/InnerSpaceCompactionTask.java | 23 +- .../compaction/execute/utils/CompactionUtils.java | 17 +- .../execute/utils/log/TsFileIdentifier.java | 3 +- .../writer/AbstractCrossCompactionWriter.java | 3 +- .../writer/AbstractInnerCompactionWriter.java | 4 +- .../compaction/schedule/CompactionTaskManager.java | 5 +- .../utils/CrossSpaceCompactionCandidate.java | 6 +- .../engine/memtable/AlignedWritableMemChunk.java | 19 +- .../iotdb/db/engine/snapshot/SnapshotLoader.java | 8 +- .../iotdb/db/engine/snapshot/SnapshotTaker.java | 2 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 131 ++++---- .../engine/storagegroup/TsFileNameGenerator.java | 9 +- .../db/engine/storagegroup/TsFileProcessor.java | 15 +- .../db/engine/storagegroup/TsFileResource.java | 187 ++++++++--- .../iotdb/db/engine/upgrade/UpgradeTask.java | 6 +- .../view/InsertNonWritableViewException.java} | 19 +- .../db/metadata/cache/DataNodeSchemaCache.java | 22 +- .../cache/DeviceUsingTemplateSchemaCache.java | 7 + .../iotdb/db/metadata/cache/SchemaCacheEntry.java | 9 + .../db/metadata/cache/TimeSeriesSchemaCache.java | 71 ++++- .../db/metadata/mnode/mem/info/BasicMNodeInfo.java | 2 +- .../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 6 - .../schemaregion/result/ShowTimeSeriesResult.java | 4 +- .../schemaregion/SchemaRegionMemoryImpl.java | 8 +- .../apache/iotdb/db/metadata/tag/TagManager.java | 3 +- .../iotdb/db/metadata/view/ViewPathType.java | 11 +- .../apache/iotdb/db/metadata/view/ViewPaths.java | 97 ++++++ .../db/mpp/common/header/ColumnHeaderConstant.java | 2 +- .../mpp/common/schematree/ClusterSchemaTree.java | 43 +++ .../common/schematree/IMeasurementSchemaInfo.java | 3 + .../common/schematree/MeasurementSchemaInfo.java | 9 + .../schematree/node/SchemaMeasurementNode.java | 8 + .../fragment/FragmentInstanceExecution.java | 14 +- .../fragment/FragmentInstanceManager.java | 11 +- .../operator/schema/SchemaQueryScanOperator.java | 1 + .../operator/schema/source/DeviceSchemaSource.java | 5 + .../operator/schema/source/ISchemaSource.java | 10 + .../schema/source/LogicalViewSchemaSource.java | 107 ++++++- .../operator/schema/source/NodeSchemaSource.java | 5 + .../schema/source/PathsUsingTemplateSource.java | 5 + .../schema/source/TimeSeriesSchemaSource.java | 114 ++++++- .../org/apache/iotdb/db/mpp/plan/Coordinator.java | 2 +- .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 164 ++++++---- .../analyze/schema/ClusterSchemaFetchExecutor.java | 19 ++ .../plan/analyze/schema/ISchemaComputation.java | 39 +++ .../mpp/plan/analyze/schema/ISchemaValidation.java | 9 + .../plan/analyze/schema/NormalSchemaFetcher.java | 206 ++++++++++-- .../plan/execution/config/ConfigTaskVisitor.java | 16 + .../config/executor/ClusterConfigTaskExecutor.java | 163 ++++++++++ .../config/executor/IConfigTaskExecutor.java | 8 + .../config/metadata/view/AlterLogicalViewTask.java | 44 +++ .../metadata/view/RenameLogicalViewTask.java | 45 +++ .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 58 +++- .../db/mpp/plan/planner/LogicalPlanVisitor.java | 11 +- .../plan/node/write/InsertMultiTabletsNode.java | 7 + .../plan/planner/plan/node/write/InsertNode.java | 20 +- .../planner/plan/node/write/InsertRowsNode.java | 7 + .../plan/node/write/InsertRowsOfOneDeviceNode.java | 7 + .../iotdb/db/mpp/plan/statement/StatementType.java | 2 + .../db/mpp/plan/statement/StatementVisitor.java | 11 + .../plan/statement/crud/InsertBaseStatement.java | 93 ++++++ .../crud/InsertMultiTabletsStatement.java | 17 + .../plan/statement/crud/InsertRowStatement.java | 128 +++++++- .../crud/InsertRowsOfOneDeviceStatement.java | 21 ++ .../plan/statement/crud/InsertRowsStatement.java | 17 + .../plan/statement/crud/InsertTabletStatement.java | 135 +++++++- .../metadata/view/AlterLogicalViewStatement.java | 115 +++++++ .../metadata/view/CreateLogicalViewStatement.java | 120 ++----- .../metadata/view/RenameLogicalViewStatement.java | 73 +++++ .../{PipeLauncher.java => PipeAgentLauncher.java} | 10 +- .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 44 ++- .../SimpleConsensusProgressIndexAssigner.java | 113 +++++++ .../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 41 ++- .../db/pipe/config/PipeCollectorConstant.java | 10 + .../apache/iotdb/db/pipe/config/PipeConfig.java | 87 ----- .../core/collector/IoTDBDataRegionCollector.java | 163 ++++++++-- .../PipeHistoricalDataRegionCollector.java | 12 +- .../PipeHistoricalDataRegionFakeCollector.java | 56 ++++ .../PipeHistoricalDataRegionTsFileCollector.java | 121 ++++++- .../realtime/PipeRealtimeDataRegionCollector.java | 23 +- ...va => PipeRealtimeDataRegionFakeCollector.java} | 58 ++-- .../PipeRealtimeDataRegionHybridCollector.java | 30 +- .../PipeRealtimeDataRegionLogCollector.java | 118 +++++++ .../PipeRealtimeDataRegionTsFileCollector.java | 119 +++++++ .../realtime/assigner/DisruptorQueue.java | 5 +- .../realtime/assigner/PipeDataRegionAssigner.java | 7 +- .../listener/PipeInsertionDataNodeListener.java | 38 ++- .../matcher/CachedSchemaPatternMatcher.java | 6 +- .../impl/iotdb/v1/IoTDBThriftConnectorV1.java | 4 +- .../impl/iotdb/v1/IoTDBThriftReceiverV1.java | 19 +- .../manager/PipeConnectorSubtaskLifeCycle.java | 8 +- .../manager/PipeConnectorSubtaskManager.java | 18 +- .../iotdb/db/pipe/core/event/EnrichedEvent.java | 75 ++++- .../core/event/impl/PipeTabletInsertionEvent.java | 69 ++-- .../core/event/impl/PipeTsFileInsertionEvent.java | 71 ++++- .../event/realtime/PipeRealtimeCollectEvent.java | 63 +++- .../realtime/PipeRealtimeCollectEventFactory.java | 4 +- .../event/view/collector/PipeEventCollector.java | 6 +- .../executor/PipeAssignerSubtaskExecutor.java | 4 +- .../executor/PipeConnectorSubtaskExecutor.java | 4 +- .../executor/PipeProcessorSubtaskExecutor.java | 4 +- .../executor/PipeSubtaskExecutorManager.java | 4 +- .../execution/scheduler/PipeSubtaskScheduler.java | 9 +- .../resource/file/PipeFileResourceManager.java | 10 +- .../file/PipeHardlinkFileDirStartupCleaner.java | 60 ++++ .../pipe/resource/wal/PipeWALResourceManager.java | 12 +- .../org/apache/iotdb/db/pipe/task/PipeBuilder.java | 28 +- .../org/apache/iotdb/db/pipe/task/PipeTask.java | 7 +- .../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 50 ++- .../task/queue/ListenableBlockingPendingQueue.java | 153 ++++++++- ... => ListenableBoundedBlockingPendingQueue.java} | 5 +- .../db/pipe/task/queue/ListenablePendingQueue.java | 159 --------- ...> ListenableUnboundedBlockingPendingQueue.java} | 9 +- .../db/pipe/task/stage/PipeTaskCollectorStage.java | 21 +- .../db/pipe/task/stage/PipeTaskConnectorStage.java | 12 +- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 17 +- .../db/pipe/task/subtask/PipeConnectorSubtask.java | 26 +- .../db/pipe/task/subtask/PipeProcessorSubtask.java | 4 +- .../iotdb/db/pipe/task/subtask/PipeSubtask.java | 20 +- .../iotdb/db/query/control/SessionManager.java | 2 + .../java/org/apache/iotdb/db/service/DataNode.java | 9 +- .../db/service/metrics/DataNodeMetricsHelper.java | 2 + .../iotdb/db/service/metrics/SystemMetrics.java | 2 +- .../org/apache/iotdb/db/utils/OpenFileNumUtil.java | 8 +- .../file/AbstractTsFileRecoverPerformer.java | 9 +- .../file/UnsealedTsFileRecoverPerformer.java | 4 + .../iotdb/db/engine/cache/ChunkCacheTest.java | 4 +- .../engine/compaction/AbstractCompactionTest.java | 6 +- .../compaction/CompactionTaskManagerTest.java | 4 +- .../db/engine/compaction/TsFileIdentifierUT.java | 16 +- .../cross/CrossSpaceCompactionSelectorTest.java | 12 +- .../CrossSpaceCompactionWithFastPerformerTest.java | 6 +- ...eCompactionWithFastPerformerValidationTest.java | 20 +- ...sSpaceCompactionWithReadPointPerformerTest.java | 6 +- ...actionWithReadPointPerformerValidationTest.java | 12 +- .../db/engine/compaction/cross/MergeTest.java | 6 +- .../cross/RewriteCompactionFileSelectorTest.java | 42 +-- .../inner/AbstractInnerSpaceCompactionTest.java | 4 +- .../inner/InnerCompactionMoreDataTest.java | 2 +- .../inner/InnerCompactionSchedulerTest.java | 6 +- .../compaction/inner/InnerCompactionTest.java | 4 +- .../SizeTieredCompactionSelectorTest.java | 2 + .../inner/sizetiered/SizeTieredCompactionTest.java | 10 +- .../recover/SizeTieredCompactionRecoverTest.java | 20 +- .../utils/MultiTsFileDeviceIteratorTest.java | 20 +- .../db/engine/snapshot/IoTDBSnapshotTest.java | 46 +-- .../engine/storagegroup/FakedTsFileResource.java | 4 +- .../iotdb/db/engine/storagegroup/TTLTest.java | 6 +- .../TsFileResourceProgressIndexTest.java | 179 +++++++++++ .../db/engine/storagegroup/TsFileResourceTest.java | 2 +- .../execution/operator/AlignedSeriesTestUtil.java | 6 +- .../collector/CachedSchemaPatternMatcherTest.java | 27 +- .../core/collector/PipeRealtimeCollectTest.java | 16 +- .../executor/PipeConnectorSubtaskExecutorTest.java | 6 +- .../executor/PipeProcessorSubtaskExecutorTest.java | 2 + .../AlignedSeriesScanLimitOffsetPushDownTest.java | 10 +- .../query/reader/series/SeriesReaderTestUtil.java | 6 +- .../series/SeriesScanLimitOffsetPushDownTest.java | 8 +- .../iotdb/db/rescon/ResourceManagerTest.java | 12 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 17 +- .../iotdb/tsfile/common/conf/TSFileConfig.java | 68 +++- .../iotdb/tsfile/fileSystem/FSFactoryProducer.java | 35 +- .../org/apache/iotdb/tsfile/fileSystem/FSPath.java | 20 +- .../org/apache/iotdb/tsfile/fileSystem/FSType.java | 3 +- .../fileInputFactory/HDFSInputFactory.java | 8 +- ...putFactory.java => HybridFileInputFactory.java} | 57 ++-- ...FSInputFactory.java => OSFileInputFactory.java} | 18 +- .../fileOutputFactory/HDFSOutputFactory.java | 8 +- ...utFactory.java => HybridFileOutputFactory.java} | 57 ++-- ...OutputFactory.java => OSFileOutputFactory.java} | 18 +- .../tsfile/fileSystem/fsFactory/FSFactory.java | 13 +- .../tsfile/fileSystem/fsFactory/HDFSFactory.java | 70 +++- .../fileSystem/fsFactory/HybridFSFactory.java | 165 ++++++++++ .../fileSystem/fsFactory/LocalFSFactory.java | 22 +- .../{HDFSFactory.java => OSFSFactory.java} | 133 +++++--- .../iotdb/tsfile/read/reader/LocalTsFileInput.java | 48 --- .../iotdb/tsfile/read/reader/TsFileInput.java | 39 +-- .../org/apache/iotdb/tsfile/utils/FSUtils.java | 153 +++++++++ .../apache/iotdb/tsfile/utils/FilePathUtils.java | 23 +- 253 files changed, 7841 insertions(+), 1968 deletions(-)
