This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch fix_last_flush_time_deadlock
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 33d4a58d987 fix ut
add f1db2923dfb Pipe: decrease the default value of
pipe_sink_selector_number to 4 for better performance (#11942)
add 85e71a43543 Fix update time index after compact aligned series (#11943)
add 8885df31144 fix PointPriorityReader calculation of aligned series null
value num (#11945)
add 9e05b262559 Add the ability for ClientManager to periodically clean up
idle objects
add 69202a94a51 reduce stack message of last time check exception for read
chunk compaction performer (#11927)
add 6f37cf83b1d [IOTDB-6293] Use nanotime in cluster heartbeat (#11916)
add 4cad6af6715 Fix compile error caused by concurrent pr (#11951)
add 3846bb7f2bc Enhance the robustness of Ratis linearizable reads for
node offline scenarios (#11954)
add 070de203710 PBTree: Decouple PageManager (#11953)
add 827735d1d77 fixed collection iotdb info unable to obtain information
(#11938)
add 15f63300fb7 Pipe: support restarting pipes on datanodes when they are
stuck (#11955)
add 14162173e73 skip insertion compaction disk space check (#11959)
add b07fafe3954 Optimized wal file deletion algorithm (#11948)
add 5c6c9727648 Enhance DataNode startup probing logic (#11957)
add 20726036498 Fix deadlock when delete timeseries after loading data.
add c52da2bdbd0 [IOTDB-6295] Adjust the timeout period in ConfigNode
registration procedure (#11962)
add 39b589381e9 Fix read chunk compaction performer null pointer exception
(#11960)
add 7f7285cf62e Load: Correctly release memory when system is in READ_ONLY
mode (#11961)
add 092fcb5860a fix npe (#11966)
add 7200e217b09 Repair unsorted file by compaction (#11887)
add 82d04599f6c Fixed issue where ConfigNode might recover incorrectly
under SimpleConsensus (#11969)
add 59fe3fee515 Add unit byte for no enough query memory
add 5e1df891725 Merge branch 'master' of github.com:apache/iotdb into
fix_last_flush_time_deadlock
add dd4e3e17459 fix concurrent issue
No new revisions were added by this update.
Summary of changes:
.../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 9 +
.../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 130 +++
.../IoTDBOrderByLimitOffsetAlignByDeviceIT.java | 1 +
.../src/assembly/resources/tools/collect-info.bat | 146 ++--
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
.../resources/conf/iotdb-confignode.properties | 5 -
.../confignode/client/DataNodeRequestType.java | 1 +
.../client/async/AsyncDataNodeClientPool.java | 5 +
.../client/async/handlers/AsyncClientHandler.java | 1 +
.../heartbeat/DataNodeHeartbeatHandler.java | 2 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 15 -
.../confignode/conf/ConfigNodeDescriptor.java | 8 -
.../statemachine/ConfigRegionStateMachine.java | 23 +
.../iotdb/confignode/manager/ConfigManager.java | 16 +-
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../manager/consensus/ConsensusManager.java | 2 -
.../confignode/manager/load/cache/LoadCache.java | 2 +-
.../manager/load/cache/node/BaseNodeCache.java | 2 +-
.../load/cache/node/ConfigNodeHeartbeatCache.java | 2 +-
.../load/cache/node/DataNodeHeartbeatCache.java | 2 +-
.../load/cache/node/NodeHeartbeatSample.java | 6 +-
.../manager/load/cache/node/NodeStatistics.java | 2 +-
.../manager/load/cache/region/RegionCache.java | 5 +-
.../load/cache/region/RegionHeartbeatSample.java | 2 +-
.../manager/load/service/HeartbeatService.java | 4 +-
.../iotdb/confignode/manager/node/NodeManager.java | 9 +
.../procedure/env/ConfigNodeProcedureEnv.java | 2 +-
.../iotdb/confignode/service/ConfigNode.java | 12 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 7 +-
.../router/priority/GreedyPriorityTest.java | 6 +-
.../priority/LeaderPriorityBalancerTest.java | 6 +-
.../manager/load/cache/LoadCacheTest.java | 2 +-
.../manager/load/cache/NodeCacheTest.java | 6 +-
.../manager/load/cache/RegionGroupCacheTest.java | 6 +-
.../manager/load/cache/RegionRouteCacheTest.java | 2 +-
.../iotdb/consensus/config/IoTConsensusConfig.java | 16 -
.../apache/iotdb/consensus/config/RatisConfig.java | 16 -
.../exception/RatisReadUnavailableException.java | 13 +-
.../iot/client/IoTConsensusClientPool.java | 2 -
.../iotdb/consensus/ratis/RatisConsensus.java | 10 +-
.../resources/conf/iotdb-datanode.properties | 5 -
.../org/apache/iotdb/db/audit/AuditLogger.java | 1 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 26 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 15 -
.../db/consensus/DataRegionConsensusImpl.java | 2 -
.../db/consensus/SchemaRegionConsensusImpl.java | 1 -
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 5 +
.../db/pipe/agent/task/PipeTaskDataNodeAgent.java | 77 ++
.../pipe/extractor/IoTDBDataRegionExtractor.java | 11 +
.../PipeRealtimeDataRegionHybridExtractor.java | 9 +-
.../iotdb/db/pipe/metric/PipeExtractorMetrics.java | 7 +-
.../subtask/connector/PipeConnectorSubtask.java | 8 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 6 +
.../protocol/client/DataNodeClientPoolFactory.java | 2 -
.../impl/DataNodeInternalRPCServiceImpl.java | 24 +
.../db/queryengine/common/MPPQueryContext.java | 10 +-
.../execution/operator/process/TopKOperator.java | 42 +-
.../execution/schedule/DriverScheduler.java | 2 +-
.../queryengine/load/LoadTsFileMemoryManager.java | 5 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 5 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 9 +-
.../plan/analyze/LoadTsfileAnalyzer.java | 9 +-
.../plan/analyze/schema/ClusterSchemaFetcher.java | 6 +-
.../plan/execution/config/ConfigTaskVisitor.java | 8 +
.../config/executor/ClusterConfigTaskExecutor.java | 49 ++
.../config/executor/IConfigTaskExecutor.java | 2 +
.../{KillQueryTask.java => RepairDataTask.java} | 12 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 12 +
.../plan/planner/LocalExecutionPlanner.java | 4 +-
.../plan/planner/LogicalPlanBuilder.java | 6 +-
.../scheduler/FragmentInstanceDispatcherImpl.java | 16 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 115 +--
.../queryengine/plan/statement/StatementType.java | 1 +
.../plan/statement/StatementVisitor.java | 5 +
...acheStatement.java => RepairDataStatement.java} | 16 +-
.../schemafile/pagemgr/BTreePageManager.java | 8 +-
.../pbtree/schemafile/pagemgr/PageIOChannel.java | 184 ++++
.../schemafile/pagemgr/PageIndexSortBuckets.java | 129 +++
.../pbtree/schemafile/pagemgr/PageManager.java | 452 +---------
.../impl/pbtree/schemafile/pagemgr/PagePool.java | 152 ++++
.../schemafile/pagemgr/SchemaPageContext.java | 109 +++
.../java/org/apache/iotdb/db/service/DataNode.java | 24 +-
.../iotdb/db/storageengine/StorageEngine.java | 45 +
.../db/storageengine/dataregion/DataRegion.java | 16 +-
.../dataregion/HashLastFlushTimeMap.java | 4 +-
.../impl/ReadChunkCompactionPerformer.java | 7 +-
.../impl/ReadPointCompactionPerformer.java | 14 +-
.../RepairUnsortedFileCompactionPerformer.java | 86 ++
.../execute/task/AbstractCompactionTask.java | 9 +-
.../execute/task/CompactionTaskPriorityType.java | 14 +-
.../execute/task/CrossSpaceCompactionTask.java | 5 +-
.../execute/task/InnerSpaceCompactionTask.java | 35 +-
.../task/InsertionCrossSpaceCompactionTask.java | 12 +-
.../task/RepairUnsortedFileCompactionTask.java | 239 +++++
.../ReadChunkAlignedSeriesCompactionExecutor.java | 11 +-
.../execute/utils/reader/PointPriorityReader.java | 2 +-
.../utils/writer/AbstractCompactionWriter.java | 6 +-
.../writer/AbstractCrossCompactionWriter.java | 6 +-
.../writer/AbstractInnerCompactionWriter.java | 6 +-
.../utils/writer/FastCrossCompactionWriter.java | 6 +-
.../utils/writer/FastInnerCompactionWriter.java | 6 +-
.../writer/ReadPointCrossCompactionWriter.java | 7 +-
.../writer/ReadPointInnerCompactionWriter.java | 7 +-
.../writer/RepairUnsortedFileCompactionWriter.java | 109 +++
.../dataregion/compaction/repair/RepairLogger.java | 136 +++
.../repair/RepairTaskRecoverLogParser.java | 115 +++
.../compaction/repair/RepairTimePartition.java | 136 +++
.../repair/UnsortedFileRepairTaskScheduler.java | 350 ++++++++
.../compaction/schedule/CompactionScheduler.java | 22 +-
.../compaction/schedule/CompactionTaskManager.java | 1 -
.../DefaultCompactionTaskComparatorImpl.java | 10 +-
... => RepairUnsortedFileCompactionEstimator.java} | 29 +-
.../impl/SizeTieredCompactionSelector.java | 40 +-
.../InsertionCrossCompactionTaskResource.java | 25 +
.../selector/utils/TsFileResourceCandidate.java | 5 +-
.../compaction/settle/SettleRequestHandler.java | 4 +-
.../dataregion/memtable/AbstractMemTable.java | 18 +
.../dataregion/memtable/IMemTable.java | 2 +
.../dataregion/memtable/TsFileProcessor.java | 5 +
.../dataregion/tsfile/TsFileManager.java | 39 +-
.../dataregion/tsfile/TsFileRepairStatus.java} | 9 +-
.../dataregion/tsfile/TsFileResource.java | 15 +
.../tsfile/generator/TsFileNameGenerator.java | 53 +-
.../dataregion/wal/buffer/WALBuffer.java | 57 +-
.../wal/checkpoint/CheckpointManager.java | 50 +-
.../dataregion/wal/checkpoint/MemTableInfo.java | 22 +-
.../dataregion/wal/io/WALByteBufReader.java | 26 +-
.../dataregion/wal/io/WALMetaData.java | 76 +-
.../storageengine/dataregion/wal/node/WALNode.java | 201 ++---
.../dataregion/wal/recover/WALNodeRecoverTask.java | 41 +-
.../TsFileOverlapValidationAndRepairTool.java | 6 -
.../AlignByTimeOrderByLimitOffsetTest.java | 5 +-
.../storageengine/dataregion/DataRegionTest.java | 4 +
.../FastInnerCompactionPerformerTest.java | 108 +++
...eCompactionWithFastPerformerValidationTest.java | 4 +-
...actionWithReadPointPerformerValidationTest.java | 2 +-
.../compaction/inner/InnerCompactionTest.java | 1 +
...nkCompactionPerformerWithAlignedSeriesTest.java | 155 +++-
.../compaction/repair/RepairLoggerTest.java | 118 +++
.../repair/RepairUnsortedFileCompactionTest.java | 973 +++++++++++++++++++++
.../utils/MultiTsFileDeviceIteratorTest.java | 12 +-
.../dataregion/wal/node/WALEntryHandlerTest.java | 13 +-
.../wal/node/WalDeleteOutdatedNewTest.java | 585 +++++++++++++
.../wal/recover/WALRecoverWriterTest.java | 9 +-
.../resources/conf/iotdb-common.properties | 25 +-
.../iotdb/commons/client/ClientPoolFactory.java | 29 +-
.../client/property/ClientPoolProperty.java | 35 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 48 +-
.../iotdb/commons/conf/CommonDescriptor.java | 33 +-
.../commons/pipe/agent/task/PipeTaskAgent.java | 4 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 20 +-
.../iotdb/commons/client/ClientManagerTest.java | 47 +-
.../src/main/thrift/confignode.thrift | 3 +
.../src/main/thrift/datanode.thrift | 2 +
156 files changed, 5202 insertions(+), 1290 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/{KillQueryTask.java
=> RepairDataTask.java} (77%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/{ClearCacheStatement.java
=> RepairDataStatement.java} (89%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageIOChannel.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageIndexSortBuckets.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PagePool.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/SchemaPageContext.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/RepairUnsortedFileCompactionWriter.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLogger.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTaskRecoverLogParser.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartition.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/UnsortedFileRepairTaskScheduler.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/{FastCrossSpaceCompactionEstimator.java
=> RepairUnsortedFileCompactionEstimator.java} (74%)
copy
iotdb-core/{confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/maintainer/RegionMaintainType.java
=>
datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileRepairStatus.java}
(85%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLoggerTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WalDeleteOutdatedNewTest.java
