This is an automated email from the ASF dual-hosted git repository. spricoder pushed a commit to branch feature/memory_transfer in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 62f94d7374159dbe2cd50b2c9fa46c846a5d95ed Merge: fbe68fd8755 fddf0a6eaa2 Author: spricoder <[email protected]> AuthorDate: Fri Mar 7 12:25:04 2025 +0800 Merge .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 73 ----- .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 17 +- .../it/auth/IoTDBClusterAuthorityRelationalIT.java | 1 - .../iotdb/db/it/auth/IoTDBRelationalAuthIT.java | 11 +- .../manual/basic/IoTDBPipeLifeCycleIT.java | 54 +--- .../manual/enhanced/IoTDBPipeIdempotentIT.java | 12 + .../it/db/it/IoTDBAuthenticationTableIT.java | 114 +++++++- .../relational/it/db/it/IoTDBInsertTableIT.java | 40 +-- .../scalar/IoTDBFormatFunctionTableIT.java | 2 +- .../query/old/query/IoTDBGreatestLeastTableIT.java | 2 +- .../it/query/recent/IoTDBMaintainAuthIT.java | 89 +++--- .../it/query/recent/IoTDBTableAggregationIT.java | 4 +- .../recent/IoTDBTableAggregationNonStreamIT.java | 56 ++++ .../relational/it/schema/IoTDBDatabaseIT.java | 53 ++-- .../iotdb/relational/it/schema/IoTDBTableIT.java | 37 ++- .../it/session/IoTDBSessionRelationalIT.java | 141 ++++++--- .../it/local/IoTDBSubscriptionBasicIT.java | 38 +-- .../iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java | 88 ++---- .../org/apache/iotdb/jdbc/IoTDBConnection.java | 7 + .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 64 +++++ .../IoTDBRelationalDatabaseMetadata.java | 94 +++++- .../base/AbstractSubscriptionConsumer.java | 7 +- iotdb-core/ainode/poetry.lock | 149 +++++++--- .../consensus/request/ConfigPhysicalPlan.java | 8 + .../consensus/request/ConfigPhysicalPlanType.java | 2 + .../request/ConfigPhysicalPlanVisitor.java | 15 + .../write/table/SetTableColumnCommentPlan.java | 62 ++++ .../request/write/table/SetTableCommentPlan.java | 57 ++++ .../iotdb/confignode/manager/ConfigManager.java | 14 + .../manager/load/balancer/PartitionBalancer.java | 8 +- .../iotdb/confignode/manager/node/NodeManager.java | 45 +-- .../extractor/ConfigRegionListeningFilter.java | 6 +- ...ConfigPhysicalPlanTablePatternParseVisitor.java | 22 ++ .../receiver/protocol/IoTDBConfigNodeReceiver.java | 118 ++++++-- .../PipeConfigPhysicalPlanTSStatusVisitor.java | 36 ++- .../manager/schema/ClusterSchemaManager.java | 40 ++- .../persistence/executor/ConfigPlanExecutor.java | 6 + .../persistence/schema/ClusterSchemaInfo.java | 39 +++ .../confignode/persistence/schema/ConfigMTree.java | 31 ++ .../request/ConfigPhysicalPlanSerDeTest.java | 33 ++- ...igPhysicalPlanTablePatternParseVisitorTest.java | 18 ++ .../apache/iotdb/consensus/pipe/PipeConsensus.java | 80 +++--- .../consensus/pipe/PipeConsensusServerImpl.java | 4 +- .../pipe/metric/PipeConsensusSyncLagManager.java | 6 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 +- .../db/consensus/DataRegionConsensusImpl.java | 2 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 6 +- .../event/realtime/PipeRealtimeEventFactory.java | 5 +- ...oricalDataRegionTsFileAndDeletionExtractor.java | 1 - .../listener/PipeInsertionDataNodeListener.java | 8 +- .../pipeconsensus/PipeConsensusReceiver.java | 2 +- .../db/pipe/resource/memory/PipeMemoryManager.java | 8 +- .../impl/DataNodeInternalRPCServiceImpl.java | 36 ++- .../queryengine/execution/memory/MemoryPool.java | 8 +- .../execution/operator/AbstractOperator.java | 5 +- .../join/SimpleNestedLoopCrossJoinOperator.java | 2 +- .../InformationSchemaContentSupplierFactory.java | 13 + .../aggregation/grouped/GroupedAggregator.java | 4 +- .../grouped/GroupedFirstAccumulator.java | 11 +- .../grouped/GroupedFirstByAccumulator.java | 2 +- .../grouped/GroupedLastAccumulator.java | 1 + .../grouped/GroupedLastByAccumulator.java | 2 +- .../grouped/GroupedVarianceAccumulator.java | 2 +- .../iotdb/db/queryengine/plan/Coordinator.java | 8 +- .../plan/analyze/ClusterPartitionFetcher.java | 5 + .../plan/analyze/IPartitionFetcher.java | 4 + .../analyze/cache/partition/PartitionCache.java | 6 +- .../cache/schema/DataNodeDevicePathCache.java | 2 +- .../schema/dualkeycache/impl/DualKeyCacheImpl.java | 9 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 4 + .../analyze/load/LoadTsFileTableSchemaCache.java | 24 +- .../analyze/schema/AutoCreateSchemaExecutor.java | 6 +- .../analyze/schema/ClusterSchemaFetchExecutor.java | 7 +- .../queryengine/plan/execution/QueryExecution.java | 4 - .../execution/config/TableConfigTaskVisitor.java | 140 ++++++--- .../config/executor/ClusterConfigTaskExecutor.java | 85 ++++++ .../config/executor/IConfigTaskExecutor.java | 16 ++ .../relational/AlterTableCommentColumnTask.java | 53 ++++ .../relational/AlterTableCommentTableTask.java | 48 ++++ .../relational/DescribeTableDetailsTask.java | 6 + .../metadata/relational/ShowTablesDetailsTask.java | 7 + .../plan/planner/LocalExecutionPlanner.java | 8 +- .../SimpleFragmentParallelPlanner.java | 2 +- .../distribution/WriteFragmentParallelPlanner.java | 3 +- .../plan/planner/plan/node/write/InsertNode.java | 3 +- .../plan/node/write/RelationalInsertRowNode.java | 5 - .../analyzer/StatementAnalyzerFactory.java | 4 + .../metadata/fetcher/TableDeviceSchemaFetcher.java | 6 +- .../fetcher/TableDeviceSchemaValidator.java | 6 +- .../fetcher/TableHeaderSchemaValidator.java | 47 ++- .../fetcher/cache/TableAttributeSchema.java | 6 +- .../fetcher/cache/TableDeviceLastCache.java | 5 +- .../fetcher/cache/TableDeviceSchemaCache.java | 6 +- .../distribute/TableModelQueryFragmentPlanner.java | 2 +- .../plan/relational/security/AccessControl.java | 8 - .../relational/security/AccessControlImpl.java | 6 +- .../relational/security/AllowAllAccessControl.java | 5 - .../relational/security/TableModelPrivilege.java | 5 - .../plan/relational/sql/ast/AstVisitor.java | 12 + .../plan/relational/sql/ast/ColumnDefinition.java | 27 +- .../plan/relational/sql/ast/CreateTable.java | 8 + .../plan/relational/sql/ast/SetColumnComment.java | 116 ++++++++ .../plan/relational/sql/ast/SetProperties.java | 12 - .../plan/relational/sql/ast/SetTableComment.java | 98 +++++++ .../relational/sql/ast/ShowQueriesStatement.java} | 40 +-- .../plan/relational/sql/parser/AstBuilder.java | 29 +- .../plan/relational/sql/rewrite/ShowRewrite.java | 26 +- .../sql/rewrite/StatementRewriteFactory.java | 6 +- .../plan/relational/sql/util/SqlFormatter.java | 42 ++- .../plan/scheduler/load/LoadTsFileScheduler.java | 117 +++++--- .../rescon/MemSchemaEngineStatistics.java | 4 +- .../db/storageengine/buffer/BloomFilterCache.java | 2 +- .../iotdb/db/storageengine/buffer/ChunkCache.java | 2 +- .../buffer/TimeSeriesMetadataCache.java | 2 +- .../execute/task/CrossSpaceCompactionTask.java | 2 +- .../execute/task/InnerSpaceCompactionTask.java | 2 +- .../task/RepairUnsortedFileCompactionTask.java | 2 +- .../compaction/execute/utils/CompactionUtils.java | 12 +- .../dataregion/memtable/AbstractMemTable.java | 9 - .../memtable/AlignedReadOnlyMemChunk.java | 318 +++++++-------------- .../memtable/AlignedWritableMemChunk.java | 134 +++------ .../memtable/AlignedWritableMemChunkGroup.java | 5 - .../dataregion/memtable/IMemTable.java | 2 - .../memtable/IWritableMemChunkGroup.java | 2 - .../dataregion/memtable/ReadOnlyMemChunk.java | 16 +- .../dataregion/memtable/TsFileProcessor.java | 12 +- .../dataregion/memtable/WritableMemChunk.java | 4 +- .../dataregion/memtable/WritableMemChunkGroup.java | 8 - .../read/reader/chunk/MemAlignedChunkReader.java | 188 ++++-------- .../read/reader/chunk/MemChunkReader.java | 4 +- .../read/reader/chunk/MemPageReader.java | 6 +- .../dataregion/tsfile/TsFileResource.java | 19 ++ .../dataregion/wal/buffer/IWALBuffer.java | 10 + .../dataregion/wal/buffer/WALBuffer.java | 13 + .../storageengine/dataregion/wal/node/WALNode.java | 8 +- .../memory/LoadTsFileDataCacheMemoryBlock.java | 2 +- .../load/memory/LoadTsFileMemoryManager.java | 2 +- .../rescon/memory/PrimitiveArrayManager.java | 2 +- .../db/storageengine/rescon/memory/SystemInfo.java | 6 +- .../rescon/memory/TimePartitionManager.java | 5 +- .../rescon/memory/TsFileResourceManager.java | 4 +- .../db/utils/datastructure/AlignedTVList.java | 55 ++-- .../datastructure/FixedPriorityBlockingQueue.java | 2 +- .../MergeSortAlignedTVListIterator.java | 147 +++------- ...tIterator.java => MergeSortTVListIterator.java} | 63 ++-- .../iotdb/db/utils/datastructure/TVList.java | 4 +- .../pipe/event/TsFileInsertionEventParserTest.java | 1 - .../plan/analyze/FakePartitionFetcherImpl.java | 5 + .../plan/planner/distribution/Util.java | 5 + .../plan/planner/distribution/Util2.java | 5 + .../plan/relational/analyzer/AnalyzerTest.java | 4 +- .../plan/relational/analyzer/TSBSMetadata.java | 7 + .../plan/relational/analyzer/TestMatadata.java | 7 + .../plan/relational/planner/PlanTester.java | 9 +- .../relational/sql/parser/AuthorStatementTest.java | 5 - .../CompactionTableModelTestFileWriter.java | 14 + .../read/reader/chunk/MemChunkLoaderTest.java | 26 +- .../iotdb/commons/auth/entity/PrivilegeType.java | 2 +- .../commons/memory/AtomicLongMemoryBlock.java | 2 +- .../apache/iotdb/commons/memory/IMemoryBlock.java | 13 +- .../apache/iotdb/commons/memory/MemoryManager.java | 146 +++++----- .../schema/column/ColumnHeaderConstant.java | 7 +- .../table/AlterOrDropTableOperationType.java | 8 +- .../commons/schema/table/InformationSchema.java | 6 + .../apache/iotdb/commons/schema/table/TsTable.java | 1 + .../iotdb/commons/memory/MemoryManagerTest.java | 14 +- .../db/relational/grammar/sql/RelationalSql.g4 | 19 +- .../src/main/thrift/confignode.thrift | 1 + 168 files changed, 2775 insertions(+), 1627 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 6840e00537a,cff92d4d359..23441e86b20 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@@ -2602,8 -2602,8 +2602,8 @@@ public class IoTDBDescriptor long newSize = storageEngineMemoryManager.getTotalMemorySizeInBytes() + consensusMemoryManager.getTotalMemorySizeInBytes(); - consensusMemoryManager.setTotalAllocatedMemorySizeInBytes(0); + consensusMemoryManager.setTotalMemorySizeInBytes(0); - storageEngineMemoryManager.setTotalMemorySizeInBytesWithReload(newSize); + storageEngineMemoryManager.setTotalAllocatedMemorySizeInBytesWithReload(newSize); SystemInfo.getInstance().loadWriteMemory(); } diff --cc iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryManager.java index 2f55481a06e,03fb10d393a..489116f6848 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryManager.java @@@ -44,19 -44,13 +44,19 @@@ public class MemoryManager private final String name; /** Whether memory management is enabled */ - private final boolean enable; + private final boolean enabled; + /** The total allocate memory size in byte of memory manager */ - private long totalAllocatedMemorySizeInBytes; ++ private volatile long totalAllocatedMemorySizeInBytes; + /** The total memory size in byte of memory manager */ - private long totalMemorySizeInBytes; + private volatile long totalMemorySizeInBytes; + /** The static allocated memory size */ - private long staticAllocatedMemorySizeInBytes = 0L; ++ private volatile long staticAllocatedMemorySizeInBytes = 0L; + /** The allocated memory size */ - private long allocatedMemorySizeInBytes = 0L; + private volatile long allocatedMemorySizeInBytes = 0L; /** The parent memory manager */ private final MemoryManager parentMemoryManager; @@@ -71,17 -65,15 +71,17 @@@ public MemoryManager(long totalMemorySizeInBytes) { this.name = "Test"; this.parentMemoryManager = null; + this.totalAllocatedMemorySizeInBytes = totalMemorySizeInBytes; this.totalMemorySizeInBytes = totalMemorySizeInBytes; - this.enable = false; + this.enabled = false; } MemoryManager(String name, MemoryManager parentMemoryManager, long totalMemorySizeInBytes) { this.name = name; this.parentMemoryManager = parentMemoryManager; + this.totalAllocatedMemorySizeInBytes = totalMemorySizeInBytes; this.totalMemorySizeInBytes = totalMemorySizeInBytes; - this.enable = false; + this.enabled = false; } private MemoryManager( @@@ -250,12 -248,16 +256,19 @@@ name, (blockName, block) -> { if (block != null) { - LOGGER.warn("register memory block already exists: {}", block); + if (block.getTotalMemorySizeInBytes() != sizeInBytes) { + LOGGER.warn( + "getOrRegisterMemoryBlock failed: memory block {} already exists, " + + "it's size is {}, requested size is {}", + blockName, + block.getTotalMemorySizeInBytes(), + sizeInBytes); + } return block; } else { + if (type.equals(MemoryBlockType.STATIC)) { + staticAllocatedMemorySizeInBytes += sizeInBytes; + } allocatedMemorySizeInBytes += sizeInBytes; return new AtomicLongMemoryBlock(name, this, sizeInBytes, type); } @@@ -458,28 -458,12 +472,32 @@@ return totalMemorySizeInBytes; } + public void setTotalMemorySizeInBytes(long totalMemorySizeInBytes) { + this.totalMemorySizeInBytes = totalMemorySizeInBytes; + } + - public void setTotalMemorySizeInBytesWithReload(long totalMemorySizeInBytes) { - reAllocateMemoryAccordingToRatio((double) totalMemorySizeInBytes / this.totalMemorySizeInBytes); + public void setTotalAllocatedMemorySizeInBytes(long totalAllocatedMemorySizeInBytes) { + this.totalMemorySizeInBytes += + (totalAllocatedMemorySizeInBytes - this.totalAllocatedMemorySizeInBytes); + this.totalAllocatedMemorySizeInBytes = totalAllocatedMemorySizeInBytes; + } + + public void setTotalAllocatedMemorySizeInBytesWithReload(long totalAllocatedMemorySizeInBytes) { + long beforeTotalMemorySizeInBytes = this.totalMemorySizeInBytes; + long afterTotalMemorySizeInBytes = + this.totalMemorySizeInBytes + + totalAllocatedMemorySizeInBytes + - this.totalAllocatedMemorySizeInBytes; + reAllocateMemoryAccordingToRatio( + (double) afterTotalMemorySizeInBytes / beforeTotalMemorySizeInBytes); + } + + public void expandTotalMemorySizeInBytes(long totalMemorySizeInBytes) { + this.totalMemorySizeInBytes += totalMemorySizeInBytes; + } + + public long getTotalAllocatedMemorySizeInBytes() { + return totalAllocatedMemorySizeInBytes; } /** Get available memory size in bytes of memory manager */
