This is an automated email from the ASF dual-hosted git repository.
lancelly pushed a change to branch max_by
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 7b15f86ad7b refactor
add 39b589381e9 Fix read chunk compaction performer null pointer exception
(#11960)
add 7f7285cf62e Load: Correctly release memory when system is in READ_ONLY
mode (#11961)
add 092fcb5860a fix npe (#11966)
add 7200e217b09 Repair unsorted file by compaction (#11887)
add 82d04599f6c Fixed issue where ConfigNode might recover incorrectly
under SimpleConsensus (#11969)
add 59fe3fee515 Add unit byte for no enough query memory
add 75ce7711162 Pipe: fixed the bug that region follower tsfiles can not
record the IoTProgressIndex from region leader (#11980)
add ebbddf38918 Multiply 3 for ScanOperator to take unsequence data into
account (#11978)
add 40fc15b23bc Pipe: fix connector subtasks can not be stopped and
restarted after exception reporting by connector subtasks (#11979)
add 6c15634b1e3 Fix estimating compaction memory without acquire read lock
(#11972)
add 7d0a735f2f3 fix.create role folder when init.
add a5445177d5d Fix template device aligned properties recover unexpectly
with SimpleConsensus (#11968)
add 769a151af97 Revert "Fixed issue where ConfigNode might recover
incorrectly under SimpleConsensus" (#11987)
add e7b0ca5793b Refactor last flush time management (#11946)
add 98310822b32 Session insert request won't fail while rolling upgrade
add 7923eb9e96e Fix concurrent bug in QueryContext
add 5619156e871 Pipe: Report load tsfile points to flush point metric
timeseries in root.__system (#11976)
add ea37a1f0f65 fix repair non aligned series (#11997)
add e331eadbeaf Fix non-latest partition last flush time cannot recover
(#11999)
add 775f169afd7 Pipe: result of show pipes statement is incorrect when it
contains back quoted pipe names (#11944)
add e2f71993826 Storage Engine: optimized DataRegion recover log (#11996)
add fcae2b48400 Pipe: support reporting progress by UserDefinedEvent
(#12000)
add 67c442b8216 refactor
No new revisions were added by this update.
Summary of changes:
.../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 132 ++
.../IoTDBOrderByLimitOffsetAlignByDeviceIT.java | 1 +
.../apache/iotdb/pipe/it/IoTDBPipeSyntaxIT.java | 3 +-
.../iotdb/pipe/api/event/UserDefinedEvent.java | 58 +
.../org/apache/iotdb/isession/SessionConfig.java | 4 +
.../java/org/apache/iotdb/session/Session.java | 27 +-
.../apache/iotdb/session/SessionConnection.java | 660 ++++++++--
.../org/apache/iotdb/session/pool/SessionPool.java | 1289 +++++++++-----------
.../iotdb/session/SessionConnectionTest.java | 9 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
.../confignode/client/DataNodeRequestType.java | 1 +
.../client/async/AsyncDataNodeClientPool.java | 5 +
.../client/async/handlers/AsyncClientHandler.java | 1 +
.../iotdb/confignode/manager/ConfigManager.java | 8 +
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../iotdb/confignode/manager/node/NodeManager.java | 9 +
.../thrift/ConfigNodeRPCServiceProcessor.java | 5 +
.../org/apache/iotdb/db/audit/AuditLogger.java | 1 +
.../IoTConsensusDataRegionStateMachine.java | 2 +-
.../schemaregion/SchemaExecutionVisitor.java | 1 -
.../apache/iotdb/db/pipe/event/EnrichedEvent.java | 4 +
.../db/pipe/event/UserDefinedEnrichedEvent.java | 86 ++
.../PipeHistoricalDataRegionTsFileExtractor.java | 10 +-
.../subtask/connector/PipeConnectorSubtask.java | 21 +-
.../connector/PipeConnectorSubtaskLifeCycle.java | 42 +-
.../subtask/processor/PipeProcessorSubtask.java | 12 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 6 +
.../impl/DataNodeInternalRPCServiceImpl.java | 24 +
.../execution/fragment/QueryContext.java | 26 +-
.../execution/load/LoadTsFileManager.java | 33 +-
.../execution/operator/process/TopKOperator.java | 42 +-
.../AbstractSeriesAggregationScanOperator.java | 2 +-
.../operator/source/AlignedSeriesScanOperator.java | 4 +-
.../operator/source/SeriesScanOperator.java | 3 +-
.../execution/schedule/DriverScheduler.java | 2 +-
.../queryengine/load/LoadTsFileMemoryManager.java | 5 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 11 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 9 +-
.../plan/analyze/LoadTsfileAnalyzer.java | 9 +-
.../plan/execution/config/ConfigTaskVisitor.java | 8 +
.../config/executor/ClusterConfigTaskExecutor.java | 52 +-
.../config/executor/IConfigTaskExecutor.java | 2 +
.../{KillQueryTask.java => RepairDataTask.java} | 12 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 24 +-
.../plan/planner/LocalExecutionPlanner.java | 4 +-
.../plan/planner/LogicalPlanBuilder.java | 6 +-
.../plan/scheduler/AsyncSendPlanNodeHandler.java | 4 +-
.../scheduler/FragmentInstanceDispatcherImpl.java | 38 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 148 ++-
.../queryengine/plan/statement/StatementType.java | 1 +
.../plan/statement/StatementVisitor.java | 5 +
...acheStatement.java => RepairDataStatement.java} | 16 +-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 1 +
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 1 +
.../mtree/impl/pbtree/CachedMTreeStore.java | 2 +-
.../pbtree/schemafile/pagemgr/PageIOChannel.java | 2 +-
.../iotdb/db/storageengine/StorageEngine.java | 45 +
.../db/storageengine/dataregion/DataRegion.java | 141 +--
.../dataregion/DeviceLastFlushTime.java} | 39 +-
.../dataregion/HashLastFlushTimeMap.java | 92 +-
.../storageengine/dataregion/ILastFlushTime.java} | 11 +-
.../dataregion/ILastFlushTimeMap.java | 12 +-
.../dataregion/PartitionLastFlushTime.java} | 26 +-
.../impl/ReadPointCompactionPerformer.java | 14 +-
.../RepairUnsortedFileCompactionPerformer.java | 86 ++
.../execute/task/AbstractCompactionTask.java | 9 +-
.../execute/task/CompactionTaskPriorityType.java | 14 +-
.../execute/task/CrossSpaceCompactionTask.java | 5 +-
.../execute/task/InnerSpaceCompactionTask.java | 35 +-
.../task/InsertionCrossSpaceCompactionTask.java | 7 +-
.../task/RepairUnsortedFileCompactionTask.java | 239 ++++
.../ReadChunkAlignedSeriesCompactionExecutor.java | 1 -
.../utils/writer/AbstractCompactionWriter.java | 6 +-
.../writer/AbstractCrossCompactionWriter.java | 6 +-
.../writer/AbstractInnerCompactionWriter.java | 6 +-
.../utils/writer/FastCrossCompactionWriter.java | 6 +-
.../utils/writer/FastInnerCompactionWriter.java | 6 +-
.../writer/ReadPointCrossCompactionWriter.java | 7 +-
.../writer/ReadPointInnerCompactionWriter.java | 7 +-
.../writer/RepairUnsortedFileCompactionWriter.java | 111 ++
.../dataregion/compaction/repair/RepairLogger.java | 136 +++
.../repair/RepairTaskRecoverLogParser.java | 115 ++
.../compaction/repair/RepairTimePartition.java | 136 +++
.../repair/UnsortedFileRepairTaskScheduler.java | 350 ++++++
.../compaction/schedule/CompactionScheduler.java | 22 +-
.../compaction/schedule/CompactionTaskManager.java | 1 -
.../DefaultCompactionTaskComparatorImpl.java | 10 +-
.../estimator/AbstractCompactionEstimator.java | 10 +
.../estimator/AbstractCrossSpaceEstimator.java | 3 +
.../estimator/AbstractInnerSpaceEstimator.java | 19 +-
... => RepairUnsortedFileCompactionEstimator.java} | 29 +-
.../impl/SizeTieredCompactionSelector.java | 40 +-
.../selector/utils/TsFileResourceCandidate.java | 5 +-
.../compaction/settle/SettleRequestHandler.java | 4 +-
.../dataregion/flush/MemTableFlushTask.java | 23 +-
.../dataregion/tsfile/TsFileManager.java | 75 +-
.../dataregion/tsfile/TsFileRepairStatus.java} | 9 +-
.../dataregion/tsfile/TsFileResource.java | 16 +
.../tsfile/generator/TsFileNameGenerator.java | 53 +-
.../storageengine/dataregion/wal/node/WALNode.java | 11 +-
.../rescon/memory/TimePartitionInfo.java | 4 +
.../rescon/memory/TimePartitionManager.java | 25 +-
.../TsFileOverlapValidationAndRepairTool.java | 6 -
.../apache/iotdb/db/utils/ErrorHandlingUtils.java | 32 +-
.../SchemaRegionSimpleRecoverTest.java | 98 ++
.../execution/operator/OperatorMemoryTest.java | 19 +-
.../AlignByTimeOrderByLimitOffsetTest.java | 5 +-
.../dataregion/LastFlushTimeMapTest.java | 121 +-
...eCompactionWithFastPerformerValidationTest.java | 4 +-
...actionWithReadPointPerformerValidationTest.java | 2 +-
.../compaction/inner/InnerCompactionTest.java | 1 +
...nkCompactionPerformerWithAlignedSeriesTest.java | 53 +
.../compaction/repair/RepairLoggerTest.java | 118 ++
.../repair/RepairUnsortedFileCompactionTest.java | 1005 +++++++++++++++
.../utils/MultiTsFileDeviceIteratorTest.java | 12 +-
.../rescon/memory/TimePartitionManagerTest.java | 18 +
.../iotdb/commons/auth/role/BasicRoleManager.java | 1 +
.../apache/iotdb/commons/conf/CommonConfig.java | 4 +
.../apache/iotdb/commons/utils/StatusUtils.java | 52 +
.../thrift-commons/src/main/thrift/common.thrift | 1 +
.../src/main/thrift/confignode.thrift | 3 +
.../src/main/thrift/datanode.thrift | 2 +
124 files changed, 4967 insertions(+), 1525 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java
create mode 100644
iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/UserDefinedEvent.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/UserDefinedEnrichedEvent.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/{KillQueryTask.java
=> RepairDataTask.java} (77%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/{ClearCacheStatement.java
=> RepairDataStatement.java} (89%)
copy
iotdb-core/{metrics/core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBGauge.java
=>
datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DeviceLastFlushTime.java}
(52%)
copy
iotdb-core/{metrics/interface/src/main/java/org/apache/iotdb/metrics/type/IMetric.java
=>
datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTime.java}
(78%)
copy
iotdb-core/{metrics/core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBCounter.java
=>
datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PartitionLastFlushTime.java}
(60%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/RepairUnsortedFileCompactionWriter.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLogger.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTaskRecoverLogParser.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartition.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/UnsortedFileRepairTaskScheduler.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/{FastCrossSpaceCompactionEstimator.java
=> RepairUnsortedFileCompactionEstimator.java} (74%)
copy
iotdb-core/{confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/maintainer/RegionMaintainType.java
=>
datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileRepairStatus.java}
(85%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLoggerTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java
