This is an automated email from the ASF dual-hosted git repository. zyk pushed a commit to branch MemoryControl in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6647ae28ec7a3feab0b34f59461f8afeaa18d578 Merge: fc6694bf9a bb9d2ff8c7 Author: zyk990424 <[email protected]> AuthorDate: Fri Aug 12 09:59:30 2022 +0800 Merge branch 'MemoryControl' of https://github.com/apache/iotdb into MemoryControl .github/workflows/sync.yml | 20 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +- .../resources/conf/iotdb-confignode.properties | 4 +- .../assembly/resources/sbin/remove-confignode.sh | 40 +- .../assembly/resources/sbin/start-confignode.sh | 36 +- .../src/assembly/resources/sbin/stop-confignode.sh | 11 +- .../AsyncConfigNodeHeartbeatClientPool.java | 4 +- .../async/datanode/AsyncDataNodeClientPool.java | 4 +- .../datanode/AsyncDataNodeHeartbeatClientPool.java | 4 +- .../sync/datanode/SyncDataNodeClientPool.java | 4 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 43 +- .../confignode/conf/ConfigNodeDescriptor.java | 17 - .../confignode/manager/ClusterSchemaManager.java | 4 +- .../iotdb/confignode/manager/ConfigManager.java | 34 +- .../apache/iotdb/confignode/manager/IManager.java | 9 + .../iotdb/confignode/manager/NodeManager.java | 54 +- .../iotdb/confignode/manager/PartitionManager.java | 33 +- .../confignode/manager/PermissionManager.java | 2 +- .../iotdb/confignode/manager/UDFManager.java | 4 +- .../iotdb/confignode/manager/load/LoadManager.java | 44 +- .../manager/load/balancer/RegionBalancer.java | 2 +- .../manager/load/balancer/RouteBalancer.java | 6 +- .../partition/GreedyPartitionAllocator.java | 25 +- .../load/balancer/router/LazyGreedyRouter.java | 4 + .../iotdb/confignode/persistence/NodeInfo.java | 22 +- .../persistence/partition/PartitionInfo.java | 47 +- .../partition/StorageGroupPartitionTable.java | 40 +- .../procedure/env/ConfigNodeProcedureEnv.java | 4 +- .../procedure/env/DataNodeRemoveHandler.java | 6 +- .../procedure/impl/RegionMigrateProcedure.java | 5 +- .../procedure/state/RegionTransitionState.java | 1 - .../service/thrift/ConfigNodeRPCService.java | 15 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 6 +- .../iotdb/confignode/persistence/NodeInfoTest.java | 4 +- .../thrift/ConfigNodeRPCServiceProcessorTest.java | 794 +-------------------- .../org/apache/iotdb/consensus/IStateMachine.java | 15 + .../apache/iotdb/consensus/config/RatisConfig.java | 2 +- .../client/AsyncMultiLeaderServiceClient.java | 5 +- .../client/MultiLeaderConsensusClientPool.java | 5 +- .../Administration-Management/Administration.md | 68 +- .../Maintenance-Tools/Maintenance-Command.md | 192 ++--- .../Administration-Management/Administration.md | 68 +- .../Maintenance-Tools/Maintenance-Command.md | 191 +++-- grafana-plugin/backend-compile.bat | 31 + grafana-plugin/go.mod | 2 +- grafana-plugin/go.sum | 15 +- grafana-plugin/pom.xml | 54 +- integration-test/import-control.xml | 4 +- .../java/org/apache/iotdb/it/env/MppConfig.java | 7 + .../org/apache/iotdb/itbase/env/BaseConfig.java | 8 + .../db/it/IoTDBClusterPartitionTableTest.java | 308 ++++++++ .../org/apache/iotdb/db/it/IoTDBConfigNodeIT.java | 639 +++++++++++++++++ .../integration/IoTDBManageTsFileResourceIT.java | 7 +- .../sync/IoTDBSyncReceiverCollectorIT.java | 513 ------------- .../db/integration/sync/IoTDBSyncReceiverIT.java | 200 +----- .../db/integration/sync/IoTDBSyncSenderIT.java | 2 + .../db/integration/sync/TransportClientMock.java | 9 - .../db/integration/sync/TransportHandlerMock.java | 3 - node-commons/pom.xml | 5 + .../commons/client/AsyncBaseClientFactory.java | 24 +- .../iotdb/commons/client/ClientPoolFactory.java | 31 +- .../AsyncConfigNodeHeartbeatServiceClient.java | 5 +- .../async/AsyncConfigNodeIServiceClient.java | 5 +- .../async/AsyncDataNodeHeartbeatServiceClient.java | 5 +- .../async/AsyncDataNodeInternalServiceClient.java | 5 +- .../AsyncDataNodeMPPDataExchangeServiceClient.java | 5 +- .../iotdb/commons/cluster/RegionRoleType.java | 20 +- .../iotdb/commons/concurrent/ThreadName.java | 3 +- .../apache/iotdb/commons/conf/CommonConfig.java | 53 ++ .../iotdb/commons/conf/CommonDescriptor.java | 18 + .../commons/partition/DataPartitionTable.java | 21 + .../commons/partition/SeriesPartitionTable.java | 22 + .../apache/iotdb/commons/sync}/SyncConstant.java | 22 +- .../apache/iotdb/commons/sync}/SyncPathUtil.java | 24 +- .../iotdb/commons/client/ClientManagerTest.java | 4 +- .../schemaregion/rocksdb/RSchemaRegion.java | 8 + .../resources/conf/iotdb-datanode.properties | 14 +- .../apache/iotdb/db/client/ConfigNodeClient.java | 3 +- .../iotdb/db/client/DataNodeClientPoolFactory.java | 28 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 53 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 41 +- .../statemachine/DataRegionStateMachine.java | 6 + .../sync/PipeDataLoadUnbearableException.java | 25 - .../db/metadata/schemaregion/ISchemaRegion.java | 6 + .../schemaregion/SchemaRegionMemoryImpl.java | 45 ++ .../schemaregion/SchemaRegionSchemaFileImpl.java | 8 + .../iotdb/db/mpp/common/header/HeaderConstant.java | 4 +- .../mpp/common/schematree/ClusterSchemaTree.java | 15 +- .../common/schematree/DeviceGroupSchemaTree.java | 98 +++ .../db/mpp/common/schematree/DeviceSchemaInfo.java | 110 ++- .../common/schematree/MeasurementSchemaInfo.java | 53 ++ .../visitor/SchemaTreeDeviceVisitor.java | 13 +- .../db/mpp/execution/exchange/SourceHandle.java | 2 +- .../operator/process/DeviceMergeOperator.java | 34 + .../operator/process/DeviceViewOperator.java | 28 + .../operator/process/FilterAndProjectOperator.java | 119 ++- .../schema/TimeSeriesSchemaScanOperator.java | 25 +- .../mpp/plan/analyze/StandaloneSchemaFetcher.java | 231 ++---- .../execution/config/metadata/ShowRegionTask.java | 1 + .../plan/expression/multi/FunctionExpression.java | 3 + .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 3 + .../db/mpp/plan/planner/LocalExecutionPlanner.java | 6 + .../db/mpp/plan/planner/OperatorTreeGenerator.java | 56 +- .../scheduler/FixedRateFragInsStateTracker.java | 10 +- .../column/multi/MappableUDFColumnTransformer.java | 4 + .../column/ternary/TernaryColumnTransformer.java | 12 + .../dag/column/unary/UnaryColumnTransformer.java | 4 + .../handler/PhysicalPlanValidationHandler.java | 4 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 55 +- .../iotdb/db/qp/physical/sys/CreatePipePlan.java | 2 +- .../db/qp/physical/sys/CreatePipeSinkPlan.java | 2 +- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 3 + .../iotdb/db/rescon/TsFileResourceManager.java | 3 +- .../iotdb/db/sync/common/ISyncInfoFetcher.java | 69 ++ .../iotdb/db/sync/common/LocalSyncInfoFetcher.java | 186 +++++ .../org/apache/iotdb/db/sync/common/SyncInfo.java | 292 ++++++++ .../db/sync/common/persistence/SyncLogReader.java | 186 +++++ .../db/sync/common/persistence/SyncLogWriter.java | 147 ++++ .../iotdb/db/sync/datasource/AbstractOpBlock.java | 16 +- .../iotdb/db/sync/datasource/DeletionGroup.java | 242 +++++++ .../iotdb/db/sync/datasource/ModsfileOpBlock.java | 53 -- .../iotdb/db/sync/datasource/PipeOpManager.java | 6 +- .../iotdb/db/sync/datasource/TsFileOpBlock.java | 463 +++++++++--- .../iotdb/db/sync/externalpipe/ExtPipePlugin.java | 15 +- .../db/sync/externalpipe/ExtPipePluginManager.java | 4 +- .../iotdb/db/sync/pipedata/DeletionPipeData.java | 12 +- .../apache/iotdb/db/sync/pipedata/PipeData.java | 24 +- .../iotdb/db/sync/pipedata/SchemaPipeData.java | 12 +- .../iotdb/db/sync/pipedata/TsFilePipeData.java | 33 +- .../sync/pipedata/queue/BufferedPipeDataQueue.java | 6 +- .../sync/pipedata/queue/PipeDataQueueFactory.java | 57 -- .../iotdb/db/sync/receiver/ReceiverService.java | 195 +---- .../db/sync/receiver/collector/Collector.java | 171 ----- .../db/sync/receiver/load/DeletionLoader.java | 5 +- .../iotdb/db/sync/receiver/load/SchemaLoader.java | 8 +- .../iotdb/db/sync/receiver/load/TsFileLoader.java | 3 +- .../db/sync/receiver/manager/ReceiverManager.java | 229 ------ .../db/sync/receiver/recovery/ReceiverLog.java | 127 ---- .../receiver/recovery/ReceiverLogAnalyzer.java | 157 ---- .../iotdb/db/sync/sender/pipe/IoTDBPipeSink.java | 2 +- .../org/apache/iotdb/db/sync/sender/pipe/Pipe.java | 4 - .../manager => sender/pipe}/PipeInfo.java | 61 +- .../iotdb/db/sync/sender/pipe/TsFilePipe.java | 14 +- .../iotdb/db/sync/sender/pipe/TsFilePipeInfo.java | 63 ++ .../db/sync/sender/recovery/SenderLogAnalyzer.java | 165 ----- .../db/sync/sender/recovery/SenderLogger.java | 141 ---- .../db/sync/sender/recovery/TsFilePipeLogger.java | 4 +- .../iotdb/db/sync/sender/service/MsgManager.java | 114 --- .../db/sync/sender/service/SenderService.java | 377 +++++----- .../db/sync/sender/service/TransportHandler.java | 56 +- .../db/sync/transport/client/ClientWrapper.java | 4 +- .../db/sync/transport/client/ITransportClient.java | 10 +- ...rtClient.java => IoTDBSInkTransportClient.java} | 148 +--- .../db/sync/transport/conf/TransportConstant.java | 36 - .../transport/server/TransportServiceImpl.java | 50 +- .../apache/iotdb/db/utils/sync/SyncPipeUtil.java | 96 +++ .../mpp/execution/operator/OperatorMemoryTest.java | 152 ++++ .../java/org/apache/iotdb/db/qp/PlannerTest.java | 11 + .../iotdb/db/rescon/ResourceManagerTest.java | 10 +- .../db/sync/datasource/DeletionGroupTest.java | 231 ++++++ .../db/sync/datasource/PipeOpManagerTest.java | 226 +++++- .../db/sync/datasource/TsFileOpBlockTest.java | 372 +++++++++- .../sync/pipedata/BufferedPipeDataQueueTest.java | 4 +- .../iotdb/db/sync/pipedata/PipeDataTest.java | 12 +- ...{ReceiverManagerTest.java => SyncInfoTest.java} | 71 +- ...ceiverLogAnalyzerTest.java => SyncLogTest.java} | 68 +- .../db/sync/transport/TransportServiceTest.java | 197 ++--- .../apache/iotdb/db/utils/EnvironmentUtils.java | 13 +- .../datanode1conf/iotdb-datanode.properties | 1 + .../datanode2conf/iotdb-datanode.properties | 1 + .../datanode3conf/iotdb-datanode.properties | 1 + .../src/main/thrift/confignode.thrift | 15 +- thrift-sync/src/main/thrift/transport.thrift | 27 - .../tsfile/read/common/block/TsBlockBuilder.java | 16 + 174 files changed, 5831 insertions(+), 4771 deletions(-)
