This is an automated email from the ASF dual-hosted git repository.
chenyz pushed a change to branch pbtree_page_concurrency
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from c9148d155a6 Add pbtree configuration and volatile size statistics
(#11958)
add 4cc5b2ba48d Fix query result is empty in order by time align by device
add 771c096b478 Pipe: let non enriched event forever retry when exception
occurred to avoid subtask fake running status recorded at data node task agent
(#11929)
add dd48a57ddb2 Improve some descriptions of daemon-script
add 63605be3186 Fix the issue where ConfigNode reports
'ConsensusGroupAlreadyExist' error during startup in some scenarios (#11928)
add c3597b6ae12 Fixed the issue that IoTConsensus synchronization is stuck
when WAL file is corrupted (#11930)
add 0cb14994659 Enable SchemaRegion Leader Auto Balance (#11926)
add f94f99a4828 Change partial aggregation symbols for variance and
standard deviation aggregators
add 3464e95f495 Load: Support configure
load_clean_up_task_execution_delay_time_seconds of a load task (#11936)
add db7d115204a Skip broken wal files when iterate the PlanNodeIterator
(#11935)
add 4ec79560c60 fix remaining issues when upgrading to ratis 3.0.1 (#11940)
add f1db2923dfb Pipe: decrease the default value of
pipe_sink_selector_number to 4 for better performance (#11942)
add 85e71a43543 Fix update time index after compact aligned series (#11943)
add 8885df31144 fix PointPriorityReader calculation of aligned series null
value num (#11945)
add 9e05b262559 Add the ability for ClientManager to periodically clean up
idle objects
add 69202a94a51 reduce stack message of last time check exception for read
chunk compaction performer (#11927)
add 6f37cf83b1d [IOTDB-6293] Use nanotime in cluster heartbeat (#11916)
add 4cad6af6715 Fix compile error caused by concurrent pr (#11951)
add 3846bb7f2bc Enhance the robustness of Ratis linearizable reads for
node offline scenarios (#11954)
add 070de203710 PBTree: Decouple PageManager (#11953)
add 827735d1d77 fixed collection iotdb info unable to obtain information
(#11938)
add 15f63300fb7 Pipe: support restarting pipes on datanodes when they are
stuck (#11955)
add 14162173e73 skip insertion compaction disk space check (#11959)
add b07fafe3954 Optimized wal file deletion algorithm (#11948)
add 5c6c9727648 Enhance DataNode startup probing logic (#11957)
add 20726036498 Fix deadlock when delete timeseries after loading data.
add c52da2bdbd0 [IOTDB-6295] Adjust the timeout period in ConfigNode
registration procedure (#11962)
add 39b589381e9 Fix read chunk compaction performer null pointer exception
(#11960)
add 7f7285cf62e Load: Correctly release memory when system is in READ_ONLY
mode (#11961)
add 092fcb5860a fix npe (#11966)
add 7200e217b09 Repair unsorted file by compaction (#11887)
add 82d04599f6c Fixed issue where ConfigNode might recover incorrectly
under SimpleConsensus (#11969)
new 0c64aea41a3 Merge branch 'master' into pbtree_page_concurrency
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 9 +
.../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 130 +++
.../src/assembly/resources/tools/collect-info.bat | 146 ++--
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
.../resources/conf/iotdb-confignode.properties | 5 -
.../assembly/resources/sbin/daemon-confignode.sh | 18 +-
.../confignode/client/DataNodeRequestType.java | 1 +
.../client/async/AsyncDataNodeClientPool.java | 5 +
.../client/async/handlers/AsyncClientHandler.java | 1 +
.../heartbeat/DataNodeHeartbeatHandler.java | 2 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 17 +-
.../confignode/conf/ConfigNodeDescriptor.java | 8 -
.../statemachine/ConfigRegionStateMachine.java | 23 +
.../iotdb/confignode/manager/ConfigManager.java | 16 +-
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../manager/consensus/ConsensusManager.java | 2 -
.../confignode/manager/load/cache/LoadCache.java | 2 +-
.../manager/load/cache/node/BaseNodeCache.java | 2 +-
.../load/cache/node/ConfigNodeHeartbeatCache.java | 2 +-
.../load/cache/node/DataNodeHeartbeatCache.java | 2 +-
.../load/cache/node/NodeHeartbeatSample.java | 6 +-
.../manager/load/cache/node/NodeStatistics.java | 2 +-
.../manager/load/cache/region/RegionCache.java | 5 +-
.../load/cache/region/RegionHeartbeatSample.java | 2 +-
.../manager/load/service/HeartbeatService.java | 4 +-
.../iotdb/confignode/manager/node/NodeManager.java | 9 +
.../procedure/env/ConfigNodeProcedureEnv.java | 2 +-
.../iotdb/confignode/service/ConfigNode.java | 12 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 7 +-
.../router/priority/GreedyPriorityTest.java | 6 +-
.../priority/LeaderPriorityBalancerTest.java | 6 +-
.../manager/load/cache/LoadCacheTest.java | 2 +-
.../manager/load/cache/NodeCacheTest.java | 6 +-
.../manager/load/cache/RegionGroupCacheTest.java | 6 +-
.../manager/load/cache/RegionRouteCacheTest.java | 2 +-
.../iotdb/consensus/config/IoTConsensusConfig.java | 16 -
.../apache/iotdb/consensus/config/RatisConfig.java | 16 -
.../exception/RatisReadUnavailableException.java | 13 +-
.../iot/client/IoTConsensusClientPool.java | 2 -
.../consensus/iot/logdispatcher/LogDispatcher.java | 24 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 18 +-
.../resources/conf/iotdb-datanode.properties | 5 -
.../src/assembly/resources/sbin/daemon-datanode.sh | 18 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 1 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 42 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 27 +-
.../db/consensus/DataRegionConsensusImpl.java | 2 -
.../db/consensus/SchemaRegionConsensusImpl.java | 1 -
.../agent/runtime/PipePeriodicalJobExecutor.java | 1 +
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 5 +
.../db/pipe/agent/task/PipeTaskDataNodeAgent.java | 77 ++
.../pipe/extractor/IoTDBDataRegionExtractor.java | 11 +
.../PipeRealtimeDataRegionHybridExtractor.java | 9 +-
.../iotdb/db/pipe/metric/PipeExtractorMetrics.java | 7 +-
.../db/pipe/task/subtask/PipeDataNodeSubtask.java | 105 ++-
.../subtask/connector/PipeConnectorSubtask.java | 150 ++--
.../iotdb/db/protocol/client/ConfigNodeClient.java | 6 +
.../protocol/client/DataNodeClientPoolFactory.java | 2 -
.../impl/DataNodeInternalRPCServiceImpl.java | 24 +
.../db/queryengine/common/MPPQueryContext.java | 10 +-
.../execution/load/LoadTsFileManager.java | 7 +-
.../execution/operator/AggregationUtil.java | 6 +
.../queryengine/load/LoadTsFileMemoryManager.java | 5 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 5 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 9 +-
.../plan/analyze/LoadTsfileAnalyzer.java | 9 +-
.../plan/analyze/schema/ClusterSchemaFetcher.java | 6 +-
.../plan/execution/config/ConfigTaskVisitor.java | 8 +
.../config/executor/ClusterConfigTaskExecutor.java | 49 ++
.../config/executor/IConfigTaskExecutor.java | 2 +
.../{KillQueryTask.java => RepairDataTask.java} | 12 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 12 +
.../plan/planner/LogicalPlanBuilder.java | 19 +-
.../plan/planner/distribution/SourceRewriter.java | 9 +-
.../plan/parameter/AggregationDescriptor.java | 14 +-
.../scheduler/FragmentInstanceDispatcherImpl.java | 16 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 123 +--
.../queryengine/plan/statement/StatementType.java | 1 +
.../plan/statement/StatementVisitor.java | 5 +
...acheStatement.java => RepairDataStatement.java} | 16 +-
.../schemafile/pagemgr/BTreePageManager.java | 8 +-
.../pbtree/schemafile/pagemgr/PageIOChannel.java | 184 ++++
.../schemafile/pagemgr/PageIndexSortBuckets.java | 129 +++
.../pbtree/schemafile/pagemgr/PageManager.java | 452 +---------
.../impl/pbtree/schemafile/pagemgr/PagePool.java | 152 ++++
.../schemafile/pagemgr/SchemaPageContext.java | 109 +++
.../java/org/apache/iotdb/db/service/DataNode.java | 24 +-
.../iotdb/db/storageengine/StorageEngine.java | 45 +
.../db/storageengine/dataregion/DataRegion.java | 16 +-
.../impl/ReadChunkCompactionPerformer.java | 7 +-
.../impl/ReadPointCompactionPerformer.java | 14 +-
.../RepairUnsortedFileCompactionPerformer.java | 86 ++
.../execute/task/AbstractCompactionTask.java | 9 +-
.../execute/task/CompactionTaskPriorityType.java | 14 +-
.../execute/task/CrossSpaceCompactionTask.java | 5 +-
.../execute/task/InnerSpaceCompactionTask.java | 35 +-
.../task/InsertionCrossSpaceCompactionTask.java | 12 +-
.../task/RepairUnsortedFileCompactionTask.java | 239 +++++
.../ReadChunkAlignedSeriesCompactionExecutor.java | 11 +-
.../execute/utils/reader/PointPriorityReader.java | 2 +-
.../utils/writer/AbstractCompactionWriter.java | 6 +-
.../writer/AbstractCrossCompactionWriter.java | 6 +-
.../writer/AbstractInnerCompactionWriter.java | 6 +-
.../utils/writer/FastCrossCompactionWriter.java | 6 +-
.../utils/writer/FastInnerCompactionWriter.java | 6 +-
.../writer/ReadPointCrossCompactionWriter.java | 7 +-
.../writer/ReadPointInnerCompactionWriter.java | 7 +-
.../writer/RepairUnsortedFileCompactionWriter.java | 109 +++
.../dataregion/compaction/repair/RepairLogger.java | 136 +++
.../repair/RepairTaskRecoverLogParser.java | 115 +++
.../compaction/repair/RepairTimePartition.java | 136 +++
.../repair/UnsortedFileRepairTaskScheduler.java | 350 ++++++++
.../compaction/schedule/CompactionScheduler.java | 22 +-
.../compaction/schedule/CompactionTaskManager.java | 1 -
.../DefaultCompactionTaskComparatorImpl.java | 10 +-
... => RepairUnsortedFileCompactionEstimator.java} | 29 +-
.../impl/SizeTieredCompactionSelector.java | 40 +-
.../InsertionCrossCompactionTaskResource.java | 25 +
.../selector/utils/TsFileResourceCandidate.java | 5 +-
.../compaction/settle/SettleRequestHandler.java | 4 +-
.../dataregion/memtable/AbstractMemTable.java | 18 +
.../dataregion/memtable/IMemTable.java | 2 +
.../dataregion/memtable/TsFileProcessor.java | 5 +
.../dataregion/tsfile/TsFileManager.java | 39 +-
.../dataregion/tsfile/TsFileRepairStatus.java} | 9 +-
.../dataregion/tsfile/TsFileResource.java | 15 +
.../tsfile/generator/TsFileNameGenerator.java | 53 +-
.../dataregion/wal/buffer/WALBuffer.java | 57 +-
.../wal/checkpoint/CheckpointManager.java | 50 +-
.../dataregion/wal/checkpoint/MemTableInfo.java | 22 +-
.../dataregion/wal/io/WALByteBufReader.java | 26 +-
.../dataregion/wal/io/WALMetaData.java | 76 +-
.../storageengine/dataregion/wal/node/WALNode.java | 216 ++---
.../dataregion/wal/recover/WALNodeRecoverTask.java | 41 +-
.../TsFileOverlapValidationAndRepairTool.java | 6 -
.../org/apache/iotdb/db/utils/SchemaUtils.java | 27 +-
.../storageengine/dataregion/DataRegionTest.java | 4 +
.../FastInnerCompactionPerformerTest.java | 108 +++
...eCompactionWithFastPerformerValidationTest.java | 4 +-
...actionWithReadPointPerformerValidationTest.java | 2 +-
.../compaction/inner/InnerCompactionTest.java | 1 +
...nkCompactionPerformerWithAlignedSeriesTest.java | 155 +++-
.../compaction/repair/RepairLoggerTest.java | 118 +++
.../repair/RepairUnsortedFileCompactionTest.java | 973 +++++++++++++++++++++
.../utils/MultiTsFileDeviceIteratorTest.java | 12 +-
.../dataregion/wal/node/WALEntryHandlerTest.java | 13 +-
.../wal/node/WalDeleteOutdatedNewTest.java | 585 +++++++++++++
.../wal/recover/WALRecoverWriterTest.java | 9 +-
.../resources/conf/iotdb-cluster.properties | 2 +-
.../resources/conf/iotdb-common.properties | 35 +-
.../iotdb/commons/client/ClientPoolFactory.java | 29 +-
.../client/property/ClientPoolProperty.java | 35 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 48 +-
.../iotdb/commons/conf/CommonDescriptor.java | 33 +-
.../commons/pipe/agent/task/PipeTaskAgent.java | 4 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 20 +-
.../iotdb/commons/client/ClientManagerTest.java | 47 +-
.../src/main/thrift/confignode.thrift | 3 +
.../src/main/thrift/datanode.thrift | 2 +
161 files changed, 5444 insertions(+), 1447 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/{KillQueryTask.java
=> RepairDataTask.java} (77%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/{ClearCacheStatement.java
=> RepairDataStatement.java} (89%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageIOChannel.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageIndexSortBuckets.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PagePool.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/SchemaPageContext.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/RepairUnsortedFileCompactionWriter.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLogger.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTaskRecoverLogParser.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartition.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/UnsortedFileRepairTaskScheduler.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/{FastCrossSpaceCompactionEstimator.java
=> RepairUnsortedFileCompactionEstimator.java} (74%)
copy
iotdb-core/{confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/maintainer/RegionMaintainType.java
=>
datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileRepairStatus.java}
(85%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLoggerTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WalDeleteOutdatedNewTest.java
