This is an automated email from the ASF dual-hosted git repository.
rong pushed a change to branch rm-load-fix
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from d8082b07c7f fix
add 17f099e03db Fix typo in SimpleNestedLoopJoinOperator
add 5911f231b79 Pipe: Using caching to optimize the verifyTableDataType
method in the Load analysis phase (#15007)
add 9154e7c67c8 revert: page access when initialize statistics and build
TsBlock for aligned memchunk (#15013)
add 83d53722cd3 Make the error message clearer for query operator
add d412a202af6 Reject insert without field column (#15017)
add f9f0574b0f7 Fix frozen IoTConsensus sync lag (#15023)
add 8fe854752de Bump jinja2 from 3.1.5 to 3.1.6 in /iotdb-core/ainode
(#15026)
add 1ab9631cb4b Added comment for table & column
add 93f0795233d IoTConsensusV2: persistent isGeneratedByPipeConsensus
(#14938)
add 1669f503907 [Fix](query)(table model) GroupedAccumulator are not
discovered before because of stream optimize
add b8947ff54fd Delete Maintain Auth
add 809ecd4cad1 Fix (#15027)
add 23068b42bbe Add memory metrics and refact memory allocation (#14710)
add f58ba839f2e Fixed the memory leakage of TableDeviceSchemaCache & The
invalidation logic when filter by SK & The bug that last time may not be
invalidated when the measurement with largest last time is invalidated (#15018)
add fddf0a6eaa2 Fixed the metadata query & Fragment Instance & retry's
timeout
add 41ada70e171 Merge branch 'master' of https://github.com/apache/iotdb
into rm-load-fix
add 430b0a053c5 Add logs
No new revisions were added by this update.
Summary of changes:
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 73 ---
.../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 17 +-
.../it/auth/IoTDBClusterAuthorityRelationalIT.java | 1 -
.../iotdb/db/it/auth/IoTDBRelationalAuthIT.java | 11 +-
.../manual/basic/IoTDBPipeLifeCycleIT.java | 54 +-
.../manual/enhanced/IoTDBPipeIdempotentIT.java | 12 +
.../it/db/it/IoTDBAuthenticationTableIT.java | 114 +++-
.../relational/it/db/it/IoTDBInsertTableIT.java | 40 +-
.../scalar/IoTDBFormatFunctionTableIT.java | 2 +-
.../query/old/query/IoTDBGreatestLeastTableIT.java | 2 +-
.../it/query/recent/IoTDBMaintainAuthIT.java | 89 +--
.../it/query/recent/IoTDBTableAggregationIT.java | 4 +-
.../IoTDBTableAggregationNonStreamIT.java} | 27 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 53 +-
.../iotdb/relational/it/schema/IoTDBTableIT.java | 37 +-
.../it/session/IoTDBSessionRelationalIT.java | 141 +++--
.../it/local/IoTDBSubscriptionBasicIT.java | 38 +-
.../base/AbstractSubscriptionConsumer.java | 7 +-
iotdb-core/ainode/poetry.lock | 149 +++--
.../consensus/request/ConfigPhysicalPlan.java | 8 +
.../consensus/request/ConfigPhysicalPlanType.java | 2 +
.../request/ConfigPhysicalPlanVisitor.java | 15 +
...umnPlan.java => SetTableColumnCommentPlan.java} | 26 +-
...bleColumnPlan.java => SetTableCommentPlan.java} | 27 +-
.../iotdb/confignode/manager/ConfigManager.java | 14 +
.../manager/load/balancer/PartitionBalancer.java | 8 +-
.../iotdb/confignode/manager/node/NodeManager.java | 45 +-
.../extractor/ConfigRegionListeningFilter.java | 6 +-
...ConfigPhysicalPlanTablePatternParseVisitor.java | 22 +
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 118 +++-
.../PipeConfigPhysicalPlanTSStatusVisitor.java | 36 +-
.../manager/schema/ClusterSchemaManager.java | 40 +-
.../persistence/executor/ConfigPlanExecutor.java | 6 +
.../persistence/schema/ClusterSchemaInfo.java | 39 ++
.../confignode/persistence/schema/ConfigMTree.java | 31 +
.../request/ConfigPhysicalPlanSerDeTest.java | 33 +-
...igPhysicalPlanTablePatternParseVisitorTest.java | 18 +
.../iotdb/consensus/config/IoTConsensusConfig.java | 30 +-
.../apache/iotdb/consensus/iot/IoTConsensus.java | 4 +-
.../logdispatcher/IoTConsensusMemoryManager.java | 50 +-
.../apache/iotdb/consensus/pipe/PipeConsensus.java | 80 +--
.../consensus/pipe/PipeConsensusServerImpl.java | 4 +-
.../pipe/metric/PipeConsensusSyncLagManager.java | 6 +-
.../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 684 +++++++++++++++++++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 345 +----------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 301 +--------
.../db/consensus/DataRegionConsensusImpl.java | 11 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 6 +-
.../event/realtime/PipeRealtimeEventFactory.java | 5 +-
...oricalDataRegionTsFileAndDeletionExtractor.java | 1 -
.../listener/PipeInsertionDataNodeListener.java | 8 +-
.../pipeconsensus/PipeConsensusReceiver.java | 2 +-
.../db/pipe/resource/memory/PipeMemoryBlock.java | 2 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 100 +--
.../impl/DataNodeInternalRPCServiceImpl.java | 42 +-
.../execution/exchange/SharedTsBlockQueue.java | 2 +-
.../execution/exchange/sink/ShuffleSinkHandle.java | 2 +-
.../execution/exchange/sink/SinkChannel.java | 2 +-
.../execution/exchange/source/SourceHandle.java | 2 +-
.../execution/memory/LocalMemoryManager.java | 5 +-
.../queryengine/execution/memory/MemoryPool.java | 42 +-
.../execution/operator/AbstractOperator.java | 5 +-
.../join/SimpleNestedLoopCrossJoinOperator.java | 2 +-
.../InformationSchemaContentSupplierFactory.java | 13 +
.../aggregation/grouped/GroupedAggregator.java | 4 +-
.../grouped/GroupedFirstAccumulator.java | 11 +-
.../grouped/GroupedFirstByAccumulator.java | 2 +-
.../grouped/GroupedLastAccumulator.java | 1 +
.../grouped/GroupedLastByAccumulator.java | 2 +-
.../grouped/GroupedVarianceAccumulator.java | 2 +-
.../execution/schedule/DriverScheduler.java | 6 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 8 +-
.../analyze/cache/partition/PartitionCache.java | 16 +-
.../cache/schema/DataNodeDevicePathCache.java | 23 +-
.../schema/dualkeycache/impl/DualKeyCacheImpl.java | 9 +-
.../plan/analyze/load/LoadTsFileAnalyzer.java | 4 +
.../analyze/load/LoadTsFileTableSchemaCache.java | 24 +-
.../analyze/schema/AutoCreateSchemaExecutor.java | 6 +-
.../analyze/schema/ClusterSchemaFetchExecutor.java | 7 +-
.../queryengine/plan/execution/QueryExecution.java | 4 -
.../execution/config/TableConfigTaskVisitor.java | 140 +++--
.../config/executor/ClusterConfigTaskExecutor.java | 85 +++
.../config/executor/IConfigTaskExecutor.java | 16 +
...nTask.java => AlterTableCommentColumnTask.java} | 13 +-
...esTask.java => AlterTableCommentTableTask.java} | 19 +-
.../relational/DescribeTableDetailsTask.java | 6 +
.../metadata/relational/ShowTablesDetailsTask.java | 7 +
.../plan/planner/LocalExecutionPlanContext.java | 3 +-
.../plan/planner/LocalExecutionPlanner.java | 124 ++--
.../SimpleFragmentParallelPlanner.java | 2 +-
.../distribution/WriteFragmentParallelPlanner.java | 3 +-
.../plan/planner/plan/node/write/InsertNode.java | 3 +-
.../plan/node/write/RelationalInsertRowNode.java | 5 -
.../analyzer/StatementAnalyzerFactory.java | 4 +
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 6 +-
.../fetcher/TableDeviceSchemaValidator.java | 6 +-
.../fetcher/TableHeaderSchemaValidator.java | 47 +-
.../fetcher/cache/TableAttributeSchema.java | 6 +-
.../fetcher/cache/TableDeviceLastCache.java | 5 +-
.../fetcher/cache/TableDeviceSchemaCache.java | 19 +-
.../distribute/TableModelQueryFragmentPlanner.java | 2 +-
.../plan/relational/security/AccessControl.java | 8 -
.../relational/security/AccessControlImpl.java | 6 +-
.../relational/security/AllowAllAccessControl.java | 5 -
.../relational/security/TableModelPrivilege.java | 5 -
.../plan/relational/sql/ast/AstVisitor.java | 12 +
.../plan/relational/sql/ast/ColumnDefinition.java | 27 +-
.../plan/relational/sql/ast/CreateTable.java | 8 +
.../ast/{DropColumn.java => SetColumnComment.java} | 26 +-
.../plan/relational/sql/ast/SetProperties.java | 12 -
.../{CountStatement.java => SetTableComment.java} | 60 +-
...onfiguration.java => ShowQueriesStatement.java} | 18 +-
.../plan/relational/sql/parser/AstBuilder.java | 29 +-
.../plan/relational/sql/rewrite/ShowRewrite.java | 26 +-
.../sql/rewrite/StatementRewriteFactory.java | 6 +-
.../plan/relational/sql/util/SqlFormatter.java | 42 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 3 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 93 +--
.../metric/SchemaEngineCachedMetric.java | 5 +-
.../rescon/MemSchemaEngineStatistics.java | 41 +-
.../ReleaseFlushStrategySizeBasedImpl.java | 6 +-
.../db/service/metrics/CompactionMetrics.java | 2 +-
.../iotdb/db/service/metrics/WritingMetrics.java | 2 +-
.../metrics/memory/ConsensusMemoryMetrics.java | 50 +-
.../metrics/memory/GlobalMemoryMetrics.java | 82 +--
.../metrics/memory/OffHeapMemoryMetrics.java | 102 +++
.../metrics/memory/QueryEngineMemoryMetrics.java | 281 ++++++---
.../metrics/memory/SchemaEngineMemoryMetrics.java | 159 +++--
.../metrics/memory/StorageEngineMemoryMetrics.java | 361 +++++++----
.../metrics/memory/StreamEngineMemoryMetrics.java | 50 +-
.../db/storageengine/buffer/BloomFilterCache.java | 27 +-
.../iotdb/db/storageengine/buffer/ChunkCache.java | 27 +-
.../buffer/TimeSeriesMetadataCache.java | 27 +-
.../storageengine/dataregion/DataRegionInfo.java | 6 +-
.../execute/task/CrossSpaceCompactionTask.java | 2 +-
.../execute/task/InnerSpaceCompactionTask.java | 2 +-
.../task/RepairUnsortedFileCompactionTask.java | 2 +-
.../compaction/execute/utils/CompactionUtils.java | 12 +-
.../memtable/AlignedReadOnlyMemChunk.java | 318 +++-------
.../memtable/AlignedWritableMemChunk.java | 134 ++--
.../dataregion/memtable/ReadOnlyMemChunk.java | 6 +-
.../read/reader/chunk/MemAlignedChunkReader.java | 188 ++----
.../dataregion/tsfile/TsFileResource.java | 19 +
.../dataregion/wal/buffer/IWALBuffer.java | 10 +
.../dataregion/wal/buffer/WALBuffer.java | 13 +
.../storageengine/dataregion/wal/node/WALNode.java | 8 +-
.../wal/utils/MemoryControlledWALEntryQueue.java | 9 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 5 +-
.../memory/LoadTsFileAnalyzeSchemaMemoryBlock.java | 2 +-
.../memory/LoadTsFileDataCacheMemoryBlock.java | 4 +-
.../load/memory/LoadTsFileMemoryManager.java | 2 +-
.../rescon/memory/PrimitiveArrayManager.java | 16 +-
.../db/storageengine/rescon/memory/SystemInfo.java | 187 ++----
.../rescon/memory/TimePartitionManager.java | 31 +-
.../rescon/memory/TsFileResourceManager.java | 40 +-
.../db/utils/datastructure/AlignedTVList.java | 55 +-
.../datastructure/FixedPriorityBlockingQueue.java | 2 +-
.../MergeSortAlignedTVListIterator.java | 147 ++---
.../datastructure/MergeSortTVListIterator.java | 55 +-
.../utils/datastructure/PageColumnAccessInfo.java | 52 --
.../pipe/event/TsFileInsertionEventParserTest.java | 1 -
.../execution/exchange/LocalSinkChannelTest.java | 7 +-
.../exchange/MPPDataExchangeManagerTest.java | 7 +-
.../execution/exchange/SharedTsBlockQueueTest.java | 4 +-
.../execution/exchange/ShuffleSinkHandleTest.java | 4 +-
.../execution/exchange/SinkChannelTest.java | 7 +-
.../execution/exchange/SourceHandleTest.java | 15 +-
.../execution/memory/MemoryPoolTest.java | 6 +-
.../plan/relational/analyzer/AnalyzerTest.java | 4 +-
.../fetcher/cache/TableDeviceSchemaCacheTest.java | 21 +-
.../plan/relational/planner/PlanTester.java | 9 +-
.../relational/sql/parser/AuthorStatementTest.java | 5 -
.../compaction/CompactionWorkerTest.java | 16 +-
.../FastCrossCompactionPerformerTest.java | 5 +-
.../cross/CrossSpaceCompactionSelectorTest.java | 5 +-
.../CompactionTableModelTestFileWriter.java | 14 +
.../compaction/utils/CompactionClearUtils.java | 2 +-
.../compaction/utils/CompactionTaskQueueTest.java | 13 +-
.../rescon/memory/ResourceManagerTest.java | 6 +-
.../rescon/memory/TimePartitionManagerTest.java | 8 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 5 +-
.../datastructure/PrimitiveArrayManagerTest.java | 8 +-
.../iotdb/commons/auth/entity/PrivilegeType.java | 2 +-
.../iotdb/commons/concurrent/ThreadName.java | 1 +
.../iotdb/commons/conf/CommonDescriptor.java | 4 +
.../commons/memory/AtomicLongMemoryBlock.java | 152 +++++
.../apache/iotdb/commons/memory/IMemoryBlock.java | 138 +++++
.../MemoryBlockType.java} | 11 +-
.../apache/iotdb/commons/memory/MemoryConfig.java} | 26 +-
.../iotdb/commons/memory/MemoryException.java} | 52 +-
.../apache/iotdb/commons/memory/MemoryManager.java | 523 ++++++++++++++++
.../schema/column/ColumnHeaderConstant.java | 7 +-
.../table/AlterOrDropTableOperationType.java | 8 +-
.../commons/schema/table/InformationSchema.java | 6 +
.../apache/iotdb/commons/schema/table/TsTable.java | 1 +
.../apache/iotdb/commons/service/ServiceType.java | 1 +
.../iotdb/commons/memory/MemoryBlockTest.java | 67 ++
.../iotdb/commons/memory/MemoryManagerTest.java | 171 ++++++
.../db/relational/grammar/sql/RelationalSql.g4 | 19 +-
.../src/main/thrift/confignode.thrift | 1 +
200 files changed, 5080 insertions(+), 2988 deletions(-)
copy
integration-test/src/test/java/org/apache/iotdb/relational/it/query/{old/aligned/IoTDBAlignedSeriesQueryTable5IT.java
=> recent/IoTDBTableAggregationNonStreamIT.java} (61%)
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/{AbstractTableColumnPlan.java
=> SetTableColumnCommentPlan.java} (71%)
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/{AbstractTableColumnPlan.java
=> SetTableCommentPlan.java} (70%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/{AlterTableDropColumnTask.java
=> AlterTableCommentColumnTask.java} (83%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/{AlterTableSetPropertiesTask.java
=> AlterTableCommentTableTask.java} (77%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{DropColumn.java
=> SetColumnComment.java} (82%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{CountStatement.java
=> SetTableComment.java} (60%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{LoadConfiguration.java
=> ShowQueriesStatement.java} (69%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/OffHeapMemoryMetrics.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/PageColumnAccessInfo.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/AtomicLongMemoryBlock.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IMemoryBlock.java
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/{pipe/datastructure/visibility/Visibility.java
=> memory/MemoryBlockType.java} (86%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/UnSetTTLTask.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryConfig.java}
(58%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RemoveConfigNode.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryException.java}
(50%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryManager.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/memory/MemoryBlockTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/memory/MemoryManagerTest.java
