This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch MemoryControl
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 4b09715779 Add enable_query_memory_estimation and merge
time_index_memory_proportion into chunk_timeseriesmeta_free_memory_proportion
add 742cc64c85 [IOTDB-4050] Failed to wait region migrate finished (#6908)
add 39db994138 [IOTDB-4051] fix nodeid duplicated problem (#6909)
add c04767fb60 update administration user-guide (#6913)
add a8f757d6e4 disable the log in state tracker for previous debugging
(#6917)
add a2a690b3d8 [IOTDB-2952] IOTDB Grafana Plugin: Build on Windows (#6907)
add b7c4062f28 [IOTDB-4056] Fix the problem that could not find or load
main class in shell `remove-confignode.sh` (#6916)
add 87d200e54b Move clientPool factory from ConfigNode to node-commons
(#6910)
add e514211827 [IOTDB-4065] support negative timestamp (before 1970 year.)
(#6918)
add 51cf191fe1 Refine writeValueColumn() to TsBlockBuilder for common use
(#6912)
add a04a1eb62b [IOTDB-3996] Forbidden CQ in REST query API (#6922)
add 0cac2c3237 [IOTDB-4046] Enhance PipeData serialize()/deserialize()
codes (#6902)
add 67c469305f purge log instantly when take a snapshot (#6724)
add a8880b51d2 [IOTDB-4054] Add some annotations for confignode.thrift
(#6919)
add 6421f4ea77 [IOTDB-4077] DataNode report region migrate result to
FOLLOWER ConfigNode sometimes, the LEADER Confignode is still waiting (#6933)
add b63c4aba78 [IOTDB-4010] Use FilterAndProjectOperator for query that
does not contain non-mappable UDF (#6929)
add ec873773e6 Correct typos in log (#6943)
add c30bb7c00d [IOTDB-3296] ext-pipe suport .mods file (#6102)
add 6188f62c81 [IOTDB-4067] Handle null pointer in
getRegisteredNodeInfoList (#6930)
add f12e15eb03 [IOTDB-4045] Optimize DataNode query relevant interfaces
(#6944)
add 2fee7a4b6a [IOTDB-3191][IOTDB-3192] Refactoring sync module to use
AbstractSyncInfo to manage pipe metadata (#6817)
add 517dc13a63 [IOTDB-4090] Add getLatestSnapshotFiles interface in
consensus (#6948)
add 085928af01 [IOTDB-4039] Distinguish the selector thread-name of
ClientPools (#6927)
add 588bf40de3 [IOTDB-3821] add some new test cases for ConfigNodeIT
(#6906)
add f1787b83c5 [IOTDB-3938] Optimize Standalone schema fetch performance
(#6949)
add af5549d325 [IOTDB-3966] [IOTDB-4079]Show leadership when show Regions
&& fix region id duplicated (#6939)
add 4c9fa69917 [IOTDB-4058] DataPartition inheritance policy (#6931)
new b89aec4907 Merge remote-tracking branch 'origin/master' into
MemoryControl
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/sync.yml | 20 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +-
.../resources/conf/iotdb-confignode.properties | 4 +-
.../assembly/resources/sbin/remove-confignode.sh | 40 +-
.../assembly/resources/sbin/start-confignode.sh | 36 +-
.../src/assembly/resources/sbin/stop-confignode.sh | 11 +-
.../AsyncConfigNodeHeartbeatClientPool.java | 4 +-
.../async/datanode/AsyncDataNodeClientPool.java | 4 +-
.../datanode/AsyncDataNodeHeartbeatClientPool.java | 4 +-
.../sync/datanode/SyncDataNodeClientPool.java | 4 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 43 +-
.../confignode/conf/ConfigNodeDescriptor.java | 17 -
.../confignode/manager/ClusterSchemaManager.java | 4 +-
.../iotdb/confignode/manager/ConfigManager.java | 34 +-
.../apache/iotdb/confignode/manager/IManager.java | 9 +
.../iotdb/confignode/manager/NodeManager.java | 54 +-
.../iotdb/confignode/manager/PartitionManager.java | 33 +-
.../confignode/manager/PermissionManager.java | 2 +-
.../iotdb/confignode/manager/UDFManager.java | 4 +-
.../iotdb/confignode/manager/load/LoadManager.java | 44 +-
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../manager/load/balancer/RouteBalancer.java | 6 +-
.../partition/GreedyPartitionAllocator.java | 25 +-
.../load/balancer/router/LazyGreedyRouter.java | 4 +
.../iotdb/confignode/persistence/NodeInfo.java | 22 +-
.../persistence/partition/PartitionInfo.java | 47 +-
.../partition/StorageGroupPartitionTable.java | 40 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 4 +-
.../procedure/env/DataNodeRemoveHandler.java | 6 +-
.../procedure/impl/RegionMigrateProcedure.java | 5 +-
.../procedure/state/RegionTransitionState.java | 1 -
.../service/thrift/ConfigNodeRPCService.java | 15 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 6 +-
.../iotdb/confignode/persistence/NodeInfoTest.java | 4 +-
.../thrift/ConfigNodeRPCServiceProcessorTest.java | 794 +--------------------
.../org/apache/iotdb/consensus/IStateMachine.java | 15 +
.../apache/iotdb/consensus/config/RatisConfig.java | 2 +-
.../client/AsyncMultiLeaderServiceClient.java | 5 +-
.../client/MultiLeaderConsensusClientPool.java | 5 +-
.../Administration-Management/Administration.md | 68 +-
.../Maintenance-Tools/Maintenance-Command.md | 192 ++---
.../Administration-Management/Administration.md | 68 +-
.../Maintenance-Tools/Maintenance-Command.md | 191 +++--
.../backend-compile.bat | 29 +-
grafana-plugin/go.mod | 2 +-
grafana-plugin/go.sum | 15 +-
grafana-plugin/pom.xml | 54 +-
integration-test/import-control.xml | 4 +-
.../java/org/apache/iotdb/it/env/MppConfig.java | 7 +
.../org/apache/iotdb/itbase/env/BaseConfig.java | 8 +
.../db/it/IoTDBClusterPartitionTableTest.java | 308 ++++++++
.../org/apache/iotdb/db/it/IoTDBConfigNodeIT.java | 639 +++++++++++++++++
.../sync/IoTDBSyncReceiverCollectorIT.java | 513 -------------
.../db/integration/sync/IoTDBSyncReceiverIT.java | 200 +-----
.../db/integration/sync/IoTDBSyncSenderIT.java | 2 +
.../db/integration/sync/TransportClientMock.java | 9 -
.../db/integration/sync/TransportHandlerMock.java | 3 -
node-commons/pom.xml | 5 +
.../commons/client/AsyncBaseClientFactory.java | 24 +-
.../iotdb/commons/client/ClientPoolFactory.java | 31 +-
.../AsyncConfigNodeHeartbeatServiceClient.java | 5 +-
.../async/AsyncConfigNodeIServiceClient.java | 5 +-
.../async/AsyncDataNodeHeartbeatServiceClient.java | 5 +-
.../async/AsyncDataNodeInternalServiceClient.java | 5 +-
.../AsyncDataNodeMPPDataExchangeServiceClient.java | 5 +-
.../{RegionStatus.java => RegionRoleType.java} | 13 +-
.../iotdb/commons/concurrent/ThreadName.java | 3 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 53 ++
.../iotdb/commons/conf/CommonDescriptor.java | 18 +
.../commons/partition/DataPartitionTable.java | 21 +
.../commons/partition/SeriesPartitionTable.java | 22 +
.../apache/iotdb/commons/sync}/SyncConstant.java | 22 +-
.../apache/iotdb/commons/sync}/SyncPathUtil.java | 24 +-
.../iotdb/commons/client/ClientManagerTest.java | 4 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 8 +
.../apache/iotdb/db/client/ConfigNodeClient.java | 3 +-
.../iotdb/db/client/DataNodeClientPoolFactory.java | 28 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 13 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 -
.../statemachine/DataRegionStateMachine.java | 6 +
.../sync/PipeDataLoadBearableException.java | 25 -
.../sync/PipeDataLoadUnbearableException.java | 25 -
.../db/metadata/schemaregion/ISchemaRegion.java | 6 +
.../schemaregion/SchemaRegionMemoryImpl.java | 45 ++
.../schemaregion/SchemaRegionSchemaFileImpl.java | 8 +
.../iotdb/db/mpp/common/header/HeaderConstant.java | 4 +-
.../mpp/common/schematree/ClusterSchemaTree.java | 15 +-
.../common/schematree/DeviceGroupSchemaTree.java | 98 +++
.../db/mpp/common/schematree/DeviceSchemaInfo.java | 110 ++-
.../common/schematree/MeasurementSchemaInfo.java} | 33 +-
.../visitor/SchemaTreeDeviceVisitor.java | 13 +-
.../db/mpp/execution/exchange/SourceHandle.java | 2 +-
.../operator/process/FilterAndProjectOperator.java | 11 +-
.../schema/TimeSeriesSchemaScanOperator.java | 25 +-
.../mpp/plan/analyze/StandaloneSchemaFetcher.java | 231 ++----
.../execution/config/metadata/ShowRegionTask.java | 1 +
.../plan/expression/multi/FunctionExpression.java | 3 +
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 3 +
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 56 +-
.../scheduler/FixedRateFragInsStateTracker.java | 10 +-
.../handler/PhysicalPlanValidationHandler.java | 4 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 55 +-
.../iotdb/db/qp/physical/sys/CreatePipePlan.java | 2 +-
.../db/qp/physical/sys/CreatePipeSinkPlan.java | 2 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 3 +
.../iotdb/db/sync/common/ISyncInfoFetcher.java | 69 ++
.../iotdb/db/sync/common/LocalSyncInfoFetcher.java | 186 +++++
.../org/apache/iotdb/db/sync/common/SyncInfo.java | 292 ++++++++
.../db/sync/common/persistence/SyncLogReader.java | 186 +++++
.../db/sync/common/persistence/SyncLogWriter.java | 147 ++++
.../iotdb/db/sync/datasource/AbstractOpBlock.java | 16 +-
.../iotdb/db/sync/datasource/DeletionGroup.java | 242 +++++++
.../iotdb/db/sync/datasource/ModsfileOpBlock.java | 53 --
.../iotdb/db/sync/datasource/PipeOpManager.java | 6 +-
.../iotdb/db/sync/datasource/TsFileOpBlock.java | 463 +++++++++---
.../iotdb/db/sync/externalpipe/ExtPipePlugin.java | 15 +-
.../db/sync/externalpipe/ExtPipePluginManager.java | 4 +-
.../iotdb/db/sync/pipedata/DeletionPipeData.java | 12 +-
.../apache/iotdb/db/sync/pipedata/PipeData.java | 24 +-
.../iotdb/db/sync/pipedata/SchemaPipeData.java | 12 +-
.../iotdb/db/sync/pipedata/TsFilePipeData.java | 33 +-
.../sync/pipedata/queue/BufferedPipeDataQueue.java | 6 +-
.../sync/pipedata/queue/PipeDataQueueFactory.java | 57 --
.../iotdb/db/sync/receiver/ReceiverService.java | 195 +----
.../db/sync/receiver/collector/Collector.java | 171 -----
.../db/sync/receiver/load/DeletionLoader.java | 5 +-
.../iotdb/db/sync/receiver/load/SchemaLoader.java | 8 +-
.../iotdb/db/sync/receiver/load/TsFileLoader.java | 3 +-
.../db/sync/receiver/manager/ReceiverManager.java | 229 ------
.../db/sync/receiver/recovery/ReceiverLog.java | 127 ----
.../receiver/recovery/ReceiverLogAnalyzer.java | 157 ----
.../iotdb/db/sync/sender/pipe/IoTDBPipeSink.java | 2 +-
.../org/apache/iotdb/db/sync/sender/pipe/Pipe.java | 4 -
.../manager => sender/pipe}/PipeInfo.java | 61 +-
.../iotdb/db/sync/sender/pipe/TsFilePipe.java | 14 +-
.../iotdb/db/sync/sender/pipe/TsFilePipeInfo.java | 63 ++
.../db/sync/sender/recovery/SenderLogAnalyzer.java | 165 -----
.../db/sync/sender/recovery/SenderLogger.java | 141 ----
.../db/sync/sender/recovery/TsFilePipeLogger.java | 4 +-
.../iotdb/db/sync/sender/service/MsgManager.java | 114 ---
.../db/sync/sender/service/SenderService.java | 377 +++++-----
.../db/sync/sender/service/TransportHandler.java | 56 +-
.../db/sync/transport/client/ClientWrapper.java | 4 +-
.../db/sync/transport/client/ITransportClient.java | 10 +-
...rtClient.java => IoTDBSInkTransportClient.java} | 148 +---
.../db/sync/transport/conf/TransportConstant.java | 36 -
.../transport/server/TransportServiceImpl.java | 50 +-
.../apache/iotdb/db/utils/sync/SyncPipeUtil.java | 96 +++
.../java/org/apache/iotdb/db/qp/PlannerTest.java | 11 +
.../db/sync/datasource/DeletionGroupTest.java | 231 ++++++
.../db/sync/datasource/PipeOpManagerTest.java | 226 +++++-
.../db/sync/datasource/TsFileOpBlockTest.java | 372 +++++++++-
.../sync/pipedata/BufferedPipeDataQueueTest.java | 4 +-
.../iotdb/db/sync/pipedata/PipeDataTest.java | 12 +-
...{ReceiverManagerTest.java => SyncInfoTest.java} | 71 +-
...ceiverLogAnalyzerTest.java => SyncLogTest.java} | 68 +-
.../db/sync/transport/TransportServiceTest.java | 197 ++---
.../apache/iotdb/db/utils/EnvironmentUtils.java | 13 +-
.../datanode1conf/iotdb-datanode.properties | 1 +
.../datanode2conf/iotdb-datanode.properties | 1 +
.../datanode3conf/iotdb-datanode.properties | 1 +
.../src/main/thrift/confignode.thrift | 15 +-
thrift-sync/src/main/thrift/transport.thrift | 27 -
.../tsfile/read/common/block/TsBlockBuilder.java | 16 +
164 files changed, 5354 insertions(+), 4783 deletions(-)
copy server/src/assembly/resources/sbin/stop-server.bat =>
grafana-plugin/backend-compile.bat (60%)
mode change 100755 => 100644
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBClusterPartitionTableTest.java
delete mode 100644
integration/src/test/java/org/apache/iotdb/db/integration/sync/IoTDBSyncReceiverCollectorIT.java
rename
confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeClientPoolFactory.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java
(84%)
copy
node-commons/src/main/java/org/apache/iotdb/commons/cluster/{RegionStatus.java
=> RegionRoleType.java} (82%)
rename {server/src/main/java/org/apache/iotdb/db/sync/conf =>
node-commons/src/main/java/org/apache/iotdb/commons/sync}/SyncConstant.java
(82%)
rename {server/src/main/java/org/apache/iotdb/db/sync/conf =>
node-commons/src/main/java/org/apache/iotdb/commons/sync}/SyncPathUtil.java
(87%)
delete mode 100644
server/src/main/java/org/apache/iotdb/db/exception/sync/PipeDataLoadBearableException.java
delete mode 100644
server/src/main/java/org/apache/iotdb/db/exception/sync/PipeDataLoadUnbearableException.java
create mode 100644
server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/DeviceGroupSchemaTree.java
copy
server/src/main/java/org/apache/iotdb/db/{exception/metadata/DeleteFailedException.java
=> mpp/common/schematree/MeasurementSchemaInfo.java} (51%)
create mode 100644
server/src/main/java/org/apache/iotdb/db/sync/common/ISyncInfoFetcher.java
create mode 100644
server/src/main/java/org/apache/iotdb/db/sync/common/LocalSyncInfoFetcher.java
create mode 100644
server/src/main/java/org/apache/iotdb/db/sync/common/SyncInfo.java
create mode 100644
server/src/main/java/org/apache/iotdb/db/sync/common/persistence/SyncLogReader.java
create mode 100644
server/src/main/java/org/apache/iotdb/db/sync/common/persistence/SyncLogWriter.java
create mode 100644
server/src/main/java/org/apache/iotdb/db/sync/datasource/DeletionGroup.java
delete mode 100644
server/src/main/java/org/apache/iotdb/db/sync/datasource/ModsfileOpBlock.java
delete mode 100644
server/src/main/java/org/apache/iotdb/db/sync/pipedata/queue/PipeDataQueueFactory.java
delete mode 100644
server/src/main/java/org/apache/iotdb/db/sync/receiver/collector/Collector.java
delete mode 100644
server/src/main/java/org/apache/iotdb/db/sync/receiver/manager/ReceiverManager.java
delete mode 100644
server/src/main/java/org/apache/iotdb/db/sync/receiver/recovery/ReceiverLog.java
delete mode 100644
server/src/main/java/org/apache/iotdb/db/sync/receiver/recovery/ReceiverLogAnalyzer.java
rename server/src/main/java/org/apache/iotdb/db/sync/{receiver/manager =>
sender/pipe}/PipeInfo.java (60%)
create mode 100644
server/src/main/java/org/apache/iotdb/db/sync/sender/pipe/TsFilePipeInfo.java
delete mode 100644
server/src/main/java/org/apache/iotdb/db/sync/sender/recovery/SenderLogAnalyzer.java
delete mode 100644
server/src/main/java/org/apache/iotdb/db/sync/sender/recovery/SenderLogger.java
delete mode 100644
server/src/main/java/org/apache/iotdb/db/sync/sender/service/MsgManager.java
rename
server/src/main/java/org/apache/iotdb/db/sync/transport/client/{TransportClient.java
=> IoTDBSInkTransportClient.java} (73%)
delete mode 100644
server/src/main/java/org/apache/iotdb/db/sync/transport/conf/TransportConstant.java
create mode 100644
server/src/main/java/org/apache/iotdb/db/utils/sync/SyncPipeUtil.java
create mode 100644
server/src/test/java/org/apache/iotdb/db/sync/datasource/DeletionGroupTest.java
rename
server/src/test/java/org/apache/iotdb/db/sync/receiver/manager/{ReceiverManagerTest.java
=> SyncInfoTest.java} (53%)
rename
server/src/test/java/org/apache/iotdb/db/sync/receiver/recovery/{ReceiverLogAnalyzerTest.java
=> SyncLogTest.java} (59%)
