This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch beyyes/fix_leader_distribution in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2299684c93add3c64c3c6ac488c74762603efe5e Merge: 6f2aaffe9b 18fc3f391e Author: Beyyes <[email protected]> AuthorDate: Thu Nov 10 19:17:40 2022 +0800 Merge branch 'master' of github.com:apache/iotdb .gitmodules | 4 +- LICENSE | 15 +- .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 + .../confignode/client/ConfigNodeRequestType.java | 2 +- .../confignode/client/DataNodeRequestType.java | 1 + .../client/async/AsyncDataNodeClientPool.java | 7 + .../client/async/handlers/AsyncClientHandler.java | 1 + .../client/sync/SyncConfigNodeClientPool.java | 5 +- .../client/sync/SyncDataNodeClientPool.java | 98 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 75 +- .../confignode/conf/ConfigNodeDescriptor.java | 501 ++++--- .../iotdb/confignode/manager/ConsensusManager.java | 47 +- .../manager/load/balancer/RouteBalancer.java | 137 +- .../manager/load/balancer/router/mcf/MCFEdge.java | 29 +- .../balancer/router/mcf/MCFLeaderBalancer.java | 305 ++++ .../iotdb/confignode/manager/node/NodeManager.java | 13 + .../manager/partition/PartitionManager.java | 12 + .../iotdb/confignode/persistence/TriggerInfo.java | 2 +- .../iotdb/confignode/persistence/UDFInfo.java | 2 +- .../confignode/persistence/node/NodeInfo.java | 16 +- .../procedure/env/ConfigNodeProcedureEnv.java | 62 +- .../procedure/env/DataNodeRemoveHandler.java | 83 +- .../impl/node/AddConfigNodeProcedure.java | 40 +- .../impl/node/RemoveDataNodeProcedure.java | 2 + .../impl/statemachine/RegionMigrateProcedure.java | 19 +- .../procedure/state/AddConfigNodeState.java | 3 +- .../iotdb/confignode/service/ConfigNode.java | 4 - .../thrift/ConfigNodeRPCServiceProcessor.java | 8 + .../balancer/router/mcf/MCFLeaderBalancerTest.java | 216 +++ .../confignode1conf/iotdb-common.properties | 2 +- .../confignode2conf/iotdb-common.properties | 2 +- .../confignode3conf/iotdb-common.properties | 2 +- .../org/apache/iotdb/consensus/IStateMachine.java | 12 + .../multileader/logdispatcher/LogDispatcher.java | 24 +- .../logdispatcher/MultiLeaderMemoryManager.java | 40 +- .../ratis/ApplicationStateMachineProxy.java | 3 +- .../iotdb/consensus/ratis/RatisConsensus.java | 23 +- .../iotdb/consensus/ratis/RatisConsensusTest.java | 18 + docs/UserGuide/Cluster/Cluster-Setup.md | 2 +- .../Sync-Tool.md | 0 .../Frequently-asked-questions.md | 0 .../Maintenance-Tools/Maintenance-Command.md | 23 + .../UserGuide/{Alert => Monitor-Alert}/Alerting.md | 0 .../Metric-Tool.md | 0 docs/UserGuide/Process-Data/Triggers.md | 23 +- docs/zh/UserGuide/Cluster/Cluster-Setup.md | 2 +- .../Sync-Tool.md | 0 .../Frequently-asked-questions.md | 0 .../Maintenance-Tools/Maintenance-Command.md | 22 + .../UserGuide/{Alert => Monitor-Alert}/Alerting.md | 0 .../Metric-Tool.md | 0 docs/zh/UserGuide/Process-Data/Triggers.md | 22 +- example/trigger/pom.xml | 2 +- example/udf/pom.xml | 3 +- grafana-plugin/README.md | 128 -- grafana-plugin/pkg/main.go | 2 +- .../pkg/plugin/iotdb_resource_handler.go | 170 +++ grafana-plugin/pkg/plugin/plugin.go | 119 +- grafana-plugin/src/ConfigEditor.tsx | 70 +- grafana-plugin/src/datasource.ts | 47 +- grafana-plugin/src/img/addIoTDBDataSource.png | Bin 0 -> 43760 bytes grafana-plugin/src/img/showData.png | Bin 0 -> 170827 bytes grafana-plugin/src/plugin.json | 4 +- grafana-plugin/src/types.ts | 3 +- .../java/org/apache/iotdb/it/env/AbstractEnv.java | 1 + .../java/org/apache/iotdb/it/env/MppConfig.java | 7 + .../org/apache/iotdb/itbase/env/BaseConfig.java | 8 + .../it/IoTDBClusterRegionLeaderBalancingIT.java | 151 ++ .../db/it/aligned/IoTDBInsertAlignedValues2IT.java | 4 +- .../db/it/aligned/IoTDBInsertAlignedValues3IT.java | 4 +- .../db/it/aligned/IoTDBInsertAlignedValues4IT.java | 4 +- .../db/it/aligned/IoTDBInsertAlignedValuesIT.java | 4 +- .../iotdb/db/it/query/IoTDBNullOperandIT.java | 242 ++++ .../withoutNull/IoTDBWithoutNullAllFilterIT.java | 39 +- .../withoutNull/IoTDBWithoutNullAnyFilterIT.java | 55 +- .../IoTDBLoadExternalTsfileWithVirtualSGIT.java | 4 +- .../db/integration/IoTDBRemovePartitionIT.java | 622 ++++---- .../resources/conf/iotdb-common.properties | 70 +- .../iotdb/commons/conf/CommonDescriptor.java | 61 +- .../apache/iotdb/commons/conf/IoTDBConstant.java | 3 +- .../iotdb/commons/partition/DataPartition.java | 8 - .../apache/iotdb/commons/path/PathPatternNode.java | 58 +- .../apache/iotdb/commons/path/PathPatternTree.java | 16 +- .../apache/iotdb/commons/trigger/TriggerTable.java | 4 +- .../iotdb/commons/path/PathPatternTreeTest.java | 19 + pom.xml | 8 +- .../apache/iotdb/db/client/ConfigNodeClient.java | 5 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 57 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 145 +- .../statemachine/DataRegionStateMachine.java | 16 + .../org/apache/iotdb/db/engine/StorageEngine.java | 1011 +------------ .../apache/iotdb/db/engine/StorageEngineV2.java | 42 +- .../engine/compaction/CompactionTaskManager.java | 1 + .../sizetiered/SizeTieredCompactionSelector.java | 78 +- .../inner/utils/MultiTsFileDeviceIterator.java | 4 +- .../iotdb/db/engine/load/AlignedChunkData.java | 339 ++--- .../org/apache/iotdb/db/engine/load/ChunkData.java | 25 +- .../apache/iotdb/db/engine/load/DeletionData.java | 4 +- .../iotdb/db/engine/load/LoadTsFileManager.java | 17 +- .../iotdb/db/engine/load/NonAlignedChunkData.java | 258 ++-- .../apache/iotdb/db/engine/load/TsFileData.java | 3 +- .../load/TsFileSplitter.java} | 415 ++---- .../iotdb/db/engine/memtable/AbstractMemTable.java | 204 +-- .../engine/memtable/AlignedWritableMemChunk.java | 10 + .../memtable/AlignedWritableMemChunkGroup.java | 5 + .../apache/iotdb/db/engine/memtable/IMemTable.java | 25 +- .../db/engine/memtable/IWritableMemChunk.java | 6 +- .../db/engine/memtable/IWritableMemChunkGroup.java | 2 + .../iotdb/db/engine/memtable/WritableMemChunk.java | 9 +- .../db/engine/memtable/WritableMemChunkGroup.java | 25 +- .../iotdb/db/engine/snapshot/SnapshotTaker.java | 18 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 741 ++-------- .../db/engine/storagegroup/DataRegionInfo.java | 2 +- ...hTimeManager.java => HashLastFlushTimeMap.java} | 235 ++- ...meManager.java => IDTableLastFlushTimeMap.java} | 179 ++- ...lushTimeManager.java => ILastFlushTimeMap.java} | 34 +- .../db/engine/storagegroup/TimePartitionInfo.java | 66 + .../engine/storagegroup/TimePartitionManager.java | 165 +++ .../db/engine/storagegroup/TsFileManager.java | 6 + .../db/engine/storagegroup/TsFileProcessor.java | 172 +-- .../db/engine/storagegroup/TsFileResource.java | 6 + .../dataregion/HashVirtualPartitioner.java | 58 - .../dataregion/StorageGroupManager.java | 505 ------- .../storagegroup/timeindex/DeviceTimeIndex.java | 4 +- .../storagegroup/timeindex/FileTimeIndex.java | 6 +- .../db/metadata/idtable/entry/DeviceEntry.java | 37 +- .../schemaregion/SchemaRegionMemoryImpl.java | 4 - .../schemaregion/SchemaRegionSchemaFileImpl.java | 4 - .../apache/iotdb/db/metadata/tag/TagManager.java | 58 +- .../db/mpp/common/header/ColumnHeaderConstant.java | 21 +- .../db/mpp/common/header/DatasetHeaderFactory.java | 4 + .../mpp/common/schematree/ClusterSchemaTree.java | 3 + .../common/schematree/node/SchemaEntityNode.java | 1 + .../iotdb/db/mpp/execution/StateMachine.java | 3 - .../fragment/FragmentInstanceContext.java | 11 +- .../fragment/FragmentInstanceExecution.java | 9 +- .../fragment/FragmentInstanceFailureInfo.java | 212 +++ .../execution/fragment/FragmentInstanceInfo.java | 21 +- .../fragment/FragmentInstanceManager.java | 35 +- .../operator/process/FilterAndProjectOperator.java | 10 +- .../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 5 + .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 34 +- .../db/mpp/plan/analyze/ExpressionAnalyzer.java | 31 +- .../mpp/plan/analyze/ExpressionTypeAnalyzer.java | 12 +- .../plan/execution/config/ConfigTaskVisitor.java | 5 +- .../config/executor/ClusterConfigTaskExecutor.java | 20 +- .../config/executor/IConfigTaskExecutor.java | 3 +- .../executor/StandaloneConfigTaskExecutor.java | 3 +- ...lusterTask.java => ShowClusterDetailsTask.java} | 78 +- .../execution/config/metadata/ShowClusterTask.java | 8 +- .../iotdb/db/mpp/plan/expression/Expression.java | 5 + .../db/mpp/plan/expression/ExpressionType.java | 2 + .../db/mpp/plan/expression/leaf/NullOperand.java | 101 ++ .../plan/expression/unary/NegationExpression.java | 2 + .../visitor/ColumnTransformerVisitor.java | 17 + .../plan/expression/visitor/ExpressionVisitor.java | 5 + .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 10 + .../plan/node/load/LoadSingleTsFileNode.java | 415 +----- .../planner/plan/node/load/LoadTsFileNode.java | 19 +- .../plan/node/load/LoadTsFilePieceNode.java | 13 +- .../metedata/write/CreateMultiTimeSeriesNode.java | 81 +- .../plan/node/metedata/write/MeasurementGroup.java | 44 + .../planner/plan/node/write/InsertRowNode.java | 2 +- .../plan/node/write/InsertRowsOfOneDeviceNode.java | 12 +- .../planner/plan/node/write/InsertTabletNode.java | 2 +- .../scheduler/AbstractFragInsStateTracker.java | 33 +- .../scheduler/FixedRateFragInsStateTracker.java | 27 +- .../scheduler/load/LoadTsFileDispatcherImpl.java | 2 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 230 ++- .../statement/metadata/ShowClusterStatement.java | 10 + .../dag/column/ColumnTransformer.java | 31 + .../binary/CompareBinaryColumnTransformer.java | 7 +- .../binary/CompareEqualToColumnTransformer.java | 2 +- .../column/binary/LogicAndColumnTransformer.java | 30 + .../binary/LogicBinaryColumnTransformer.java | 24 - .../column/binary/LogicOrColumnTransformer.java | 30 + .../dag/column/leaf/NullColumnTransformer.java} | 31 +- .../ternary/CompareTernaryColumnTransformer.java | 17 +- .../dag/column/unary/InColumnTransformer.java | 8 +- .../column/unary/LogicNotColumnTransformer.java | 2 +- .../dag/column/unary/RegularColumnTransformer.java | 2 +- .../apache/iotdb/db/qp/executor/IPlanExecutor.java | 79 -- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 1374 ------------------ .../org/apache/iotdb/db/rescon/SystemInfo.java | 31 +- .../java/org/apache/iotdb/db/service/DataNode.java | 2 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 10 - .../apache/iotdb/db/service/IoTDBShutdownHook.java | 9 +- .../org/apache/iotdb/db/service/MQTTService.java | 6 +- .../org/apache/iotdb/db/service/SettleService.java | 36 +- .../org/apache/iotdb/db/service/UpgradeSevice.java | 16 +- .../impl/DataNodeInternalRPCServiceImpl.java | 34 +- .../sync/pipedata/queue/BufferedPipeDataQueue.java | 6 +- .../db/sync/receiver/load/DeletionLoader.java | 7 - .../iotdb/db/sync/receiver/load/TsFileLoader.java | 15 - .../apache/iotdb/db/tools/TsFileSketchTool.java | 193 ++- .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 10 +- .../db/tools/dataregion/DeviceMappingViewer.java | 63 - .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 6 +- .../iotdb/db/trigger/executor/TriggerExecutor.java | 3 +- .../trigger/service/TriggerManagementService.java | 8 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 24 + .../apache/iotdb/db/utils/TimePartitionUtils.java | 4 + .../db/utils/datastructure/AlignedTVList.java | 4 +- .../iotdb/db/utils/datastructure/BinaryTVList.java | 17 +- .../db/utils/datastructure/BooleanTVList.java | 13 +- .../iotdb/db/utils/datastructure/DoubleTVList.java | 13 +- .../iotdb/db/utils/datastructure/FloatTVList.java | 13 +- .../iotdb/db/utils/datastructure/IntTVList.java | 13 +- .../iotdb/db/utils/datastructure/LongTVList.java | 13 +- .../iotdb/db/utils/datastructure/TVList.java | 20 +- .../org/apache/iotdb/db/wal/buffer/WALEntry.java | 23 +- .../apache/iotdb/db/wal/buffer/WALEntryType.java | 3 + .../apache/iotdb/db/wal/buffer/WALInfoEntry.java | 18 +- .../org/apache/iotdb/db/wal/node/IWALNode.java | 11 - .../org/apache/iotdb/db/wal/node/WALFakeNode.java | 19 - .../java/org/apache/iotdb/db/wal/node/WALNode.java | 42 +- .../db/wal/recover/file/TsFilePlanRedoer.java | 73 - .../file/UnsealedTsFileRecoverPerformer.java | 11 +- .../engine/compaction/AbstractCompactionTest.java | 21 + .../engine/compaction/CompactionSchedulerTest.java | 103 +- .../ReadPointCompactionPerformerTest.java | 153 +- .../compaction/inner/InnerSeqCompactionTest.java | 6 +- .../utils/MultiTsFileDeviceIteratorTest.java | 120 ++ .../db/engine/memtable/MemTableFlushTaskTest.java | 6 +- .../db/engine/memtable/MemTableTestUtils.java | 69 +- .../db/engine/memtable/PrimitiveMemTableTest.java | 65 +- .../db/engine/storagegroup/DataRegionTest.java | 87 +- .../storagegroup/FileNodeManagerBenchmark.java | 132 -- .../storagegroup/IDTableLastFlushTimeMapTest.java | 129 ++ .../engine/storagegroup/LastFlushTimeMapTest.java | 223 +++ .../storagegroup/StorageGroupProcessorTest.java | 871 ------------ .../iotdb/db/engine/storagegroup/TTLTest.java | 156 +- .../storagegroup/TimePartitionManagerTest.java | 129 ++ .../engine/storagegroup/TsFileProcessorTest.java | 476 ------- .../engine/storagegroup/TsFileProcessorV2Test.java | 12 +- .../dataregion/HashVirtualPartitionerTest.java | 63 - .../org/apache/iotdb/db/metadata/TemplateTest.java | 137 +- .../db/metadata/idtable/IDTableFlushTimeTest.java | 226 --- .../db/metadata/idtable/IDTableRecoverTest.java | 354 +++-- .../db/metadata/idtable/LastQueryWithIDTable.java | 285 ++-- .../common/schematree/ClusterSchemaTreeTest.java | 25 + .../FragmentInstanceFailureInfoSerdeTest.java | 85 ++ .../iotdb/db/qp/physical/ConcatOptimizerTest.java | 140 -- .../qp/physical/InsertRowsOfOneDevicePlanTest.java | 75 - .../db/qp/physical/PhysicalPlanSerializeTest.java | 326 ----- .../iotdb/db/qp/physical/PhysicalPlanTest.java | 1490 -------------------- .../iotdb/db/qp/physical/SerializationTest.java | 151 -- .../iotdb/db/qp/physical/SerializedSizeTest.java | 130 -- .../iotdb/db/query/dataset/SingleDataSetTest.java | 199 ++- .../sync/pipedata/BufferedPipeDataQueueTest.java | 1 + .../iotdb/db/tools/TsFileAndModSettleToolTest.java | 397 +++--- .../org/apache/iotdb/db/tools/WalCheckerTest.java | 16 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 14 +- .../org/apache/iotdb/db/utils/MemUtilsTest.java | 35 +- .../iotdb/db/utils/TsFileRewriteToolTest.java | 11 +- .../org/apache/iotdb/db/wal/WALManagerTest.java | 52 +- .../db/wal/allocation/ElasticStrategyTest.java | 16 +- .../db/wal/allocation/FirstCreateStrategyTest.java | 18 +- .../db/wal/allocation/RoundRobinStrategyTest.java | 16 +- .../iotdb/db/wal/buffer/WALBufferCommonTest.java | 68 +- .../org/apache/iotdb/db/wal/io/WALFileTest.java | 119 +- .../apache/iotdb/db/wal/node/WALFakeNodeTest.java | 22 +- .../org/apache/iotdb/db/wal/node/WALNodeTest.java | 90 +- .../db/wal/recover/WALRecoverManagerTest.java | 100 +- .../db/wal/recover/file/TsFilePlanRedoerTest.java | 240 ++-- .../file/UnsealedTsFileRecoverPerformerTest.java | 44 +- .../datanode1conf/iotdb-common.properties | 2 +- .../datanode2conf/iotdb-common.properties | 2 +- .../datanode3conf/iotdb-common.properties | 2 +- .../datanode3conf/iotdb-datanode.properties | 2 +- .../src/test/resources/iotdb-datanode.properties | 2 - .../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 + site/src/main/.vuepress/config.js | 43 +- .../apache/iotdb/spark/db/EnvironmentUtils.java | 10 +- .../src/main/thrift/confignode.thrift | 3 + thrift/src/main/thrift/datanode.thrift | 10 +- .../iotdb/tsfile/file/header/ChunkHeader.java | 21 + .../iotdb/tsfile/file/header/PageHeader.java | 10 +- .../iotdb/tsfile/read/TsFileDeviceIterator.java | 34 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 118 +- .../iotdb/tsfile/read/common/block/TsBlock.java | 5 +- .../read/common/block/column/NullColumn.java | 87 +- .../iotdb/tsfile/utils/TsFileGeneratorUtils.java | 2 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 4 + .../write/writer/TsFileIOWriterEndFileTest.java | 49 + 287 files changed, 7974 insertions(+), 13010 deletions(-)
