This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch speed_up_recover
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from fe189239b89 try to fix 1c3d IT
add 24edb72009c fixing recover from wal not recode fileTimeIndex
add c6579d67d7a Add CI for rc branches
add 1907f4f804f Fix WAL file end marker (#13125)
add 5d0935495f5 fix ut (#13145)
add 53af23e7182 [Region Migration] Convert a Ratis exception to enable
read retry #13142
add 0cb2eb599c1 Using construction method of TimeSeriesOperand which has
type parameter
add 72f4fdc72eb Make fetch schema error more specific and support retry
for TOO_MANY_CONCURRENT_QUERIES_ERROR
add e3582cd86fd Fixed the bug that qualifiedName can not be used in device
query
add b8426f53a35 PipeConsensus: Avoid replicate block && Avoid transfer
error when connector is closed. (#13146)
add cdbac42b8b6 Fixed the bug that table compatibility check may take too
much locks in tree dialect
add ce0c090c553 Share device info in compaction schedule (#13150)
add bc85409c6cf [ISSUE-13155] When having more than one pipeline
subscription and restarting the server, the second subscription ends in a loop
of doom (#13156)
add 525f40ce3eb Fix getTsValue of LongLiteral and validation order
add ef54f795902 [Region Migration] Wait for all region-related resources
to be released before deleting (#13148)
add 34c63cf1219 Load: Support Datanode actively listening dirs and auto
load tsfile (#13085)
add be1fdde9891 Pipe: Fix ProgressReportEvent may be transferred / marked
as rate in config subtask (#13161)
add 67642d6dd37 Added check for table TTL to ensure that it's equal to or
greater than 0
add 8473124404d Support constant value in select clause
add 0840b23644c Support upgrade to v1.4 with v1.3 wal
add ba83299237a Clear Invalid schema region when restarting (#13164)
add 62e3ecf0c8c Fix select into bug while using alias
add cb4281cac55 Fix insert case sensitivity of table model
add ecd087cc238 [region migration] Ignore old region migration procedure
during procedure framework updating #13138
add 0a26801fbd1 Introduce some subquery plan code
add 6d0720c8d83 Add Cpp client CI for Ubuntu 20 and 24 (#13190)
add 1cde04af8c5 hide cli throw about history Exception (#13189)
add 798414460d5 Merge branch 'master' of github.com:apache/iotdb into
speed_up_recover
add 481e34c7397 recover from wal need to recode file timeindex cache
No new revisions were added by this update.
Summary of changes:
.github/workflows/cluster-it-1c1d.yml | 2 +
.github/workflows/cluster-it-1c3d.yml | 2 +
.github/workflows/multi-language-client.yml | 15 +-
.github/workflows/pipe-it-2cluster.yml | 2 +
.github/workflows/sonar-codecov.yml | 2 +
.github/workflows/unit-test.yml | 2 +
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 21 ++
.../relational/it/db/it/IoTDBInsertTableIT.java | 48 +++
.../it/query/recent/IoTDBNullIdQueryIT.java | 24 +-
.../relational/it/schema/IoTDBDeviceQueryIT.java | 8 +
.../iotdb/relational/it/schema/IoTDBTableIT.java | 19 ++
.../src/main/java/org/apache/iotdb/cli/Cli.java | 5 +
.../org/apache/iotdb/cli/utils/JlineUtils.java | 4 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../pipe/execution/PipeConfigNodeSubtask.java | 7 +-
.../impl/region/RegionMigrateProcedure.java | 5 +-
.../procedure/store/ProcedureFactory.java | 9 +-
.../org/apache/iotdb/consensus/IStateMachine.java | 9 +
.../apache/iotdb/consensus/iot/IoTConsensus.java | 2 +
.../consensus/iot/IoTConsensusServerImpl.java | 39 +++
.../service/IoTConsensusRPCServiceProcessor.java | 21 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 2 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 97 +++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 86 +++++-
.../dataregion/DataRegionStateMachine.java | 12 +
.../iotdb/db/exception/LoadEmptyFileException.java | 4 +-
.../metadata/AliasAlreadyExistException.java | 4 +-
.../metadata/AlignedTimeseriesException.java | 7 +-
.../metadata/DatabaseNotSetException.java | 12 +-
.../metadata/IllegalParameterOfPathException.java | 7 +-
.../metadata/MeasurementAlreadyExistException.java | 4 +-
.../metadata/PathAlreadyExistException.java | 4 +-
.../template/TemplateIncompatibleException.java | 11 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 14 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 1 +
.../execution/executor/RegionReadExecutor.java | 7 +
.../fragment/FragmentInstanceContext.java | 29 +-
.../fragment/FragmentInstanceExecution.java | 22 +-
.../execution/fragment/FragmentInstanceInfo.java | 20 ++
.../fragment/FragmentInstanceManager.java | 55 +++-
.../execution/load/LoadTsFileManager.java | 10 +
.../load/active/ActiveLoadDirScanner.java | 175 +++++++++++
.../load/active/ActiveLoadPendingQueue.java | 64 ++++
.../load/active/ActiveLoadTsFileLoader.java | 310 ++++++++++++++++++++
.../load/{ => limiter}/LoadTsFileRateLimiter.java | 2 +-
.../load/{ => splitter}/AlignedChunkData.java | 2 +-
.../execution/load/{ => splitter}/ChunkData.java | 2 +-
.../load/{ => splitter}/DeletionData.java | 2 +-
.../load/{ => splitter}/NonAlignedChunkData.java | 2 +-
.../execution/load/{ => splitter}/TsFileData.java | 2 +-
.../load/{ => splitter}/TsFileSplitter.java | 2 +-
.../relational/ColumnTransformerBuilder.java | 20 +-
.../schedule/queue/IndexedBlockingQueue.java | 5 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 11 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 166 ++++++-----
.../queryengine/plan/analyze/ExpressionUtils.java | 2 +-
...TsfileAnalyzer.java => LoadTsFileAnalyzer.java} | 23 +-
.../analyze/lock/DataNodeSchemaLockManager.java | 23 +-
.../analyze/schema/ClusterSchemaFetchExecutor.java | 7 +-
.../plan/analyze/schema/ClusterSchemaFetcher.java | 64 ++--
.../plan/analyze/schema/SchemaValidator.java | 3 +-
.../execution/config/TableConfigTaskVisitor.java | 13 +-
.../plan/node/load/LoadTsFilePieceNode.java | 2 +-
.../relational/analyzer/StatementAnalyzer.java | 4 +-
.../plan/relational/metadata/MetadataUtil.java | 3 +-
.../fetcher/TableHeaderSchemaValidator.java | 6 +-
.../plan/relational/planner/LogicalPlanner.java | 3 +-
.../plan/relational/planner/PlanBuilder.java | 24 +-
.../plan/relational/planner/QueryPlanner.java | 57 +++-
.../plan/relational/planner/RelationPlanner.java | 18 +-
.../plan/relational/planner/SubqueryPlanner.java | 322 +++++++++++++++++++++
.../relational/planner/ir/ExpressionRewriter.java | 9 +-
.../planner/ir/ExpressionTreeRewriter.java | 8 +
.../relational/sql/ast/AbstractQueryDevice.java | 22 +-
.../plan/relational/sql/ast/CountDevice.java | 4 +-
.../plan/relational/sql/ast/InsertRow.java | 10 +-
.../plan/relational/sql/ast/InsertRows.java | 20 +-
.../plan/relational/sql/ast/InsertTablet.java | 9 +-
.../plan/relational/sql/ast/LongLiteral.java | 2 +-
.../plan/relational/sql/ast/ShowDevice.java | 4 +-
.../relational/sql/ast/WrappedInsertStatement.java | 4 +
.../plan/relational/sql/parser/AstBuilder.java | 16 +-
.../scheduler/AbstractFragInsStateTracker.java | 3 +-
.../plan/scheduler/ClusterScheduler.java | 4 +-
.../scheduler/FixedRateFragInsStateTracker.java | 6 +
.../scheduler/FragmentInstanceDispatcherImpl.java | 8 +
.../plan/scheduler/load/LoadTsFileScheduler.java | 6 +-
.../plan/statement/crud/InsertBaseStatement.java | 24 ++
.../crud/InsertMultiTabletsStatement.java | 28 ++
.../crud/InsertRowsOfOneDeviceStatement.java | 28 ++
.../plan/statement/crud/InsertRowsStatement.java | 28 ++
.../java/org/apache/iotdb/db/service/DataNode.java | 59 ++--
.../db/service/metrics/CompactionMetrics.java | 14 +-
.../iotdb/db/storageengine/StorageEngine.java | 9 +-
.../db/storageengine/dataregion/DataRegion.java | 59 ++--
.../utils/CompactionTableSchemaCollector.java | 2 +-
.../compaction/io/CompactionTsFileWriter.java | 22 +-
...Summary.java => CompactionScheduleContext.java} | 28 +-
.../compaction/schedule/CompactionScheduler.java | 47 +--
.../constant/InnerSequenceCompactionSelector.java | 4 +-
.../InnerUnsequenceCompactionSelector.java | 4 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 43 ++-
.../utils/CrossSpaceCompactionCandidate.java | 26 +-
.../selector/utils/TsFileResourceCandidate.java | 13 +-
.../dataregion/memtable/AbstractMemTable.java | 47 +++
.../dataregion/memtable/TsFileProcessor.java | 3 +-
.../storageengine/dataregion/tsfile/TsFileID.java | 6 +-
.../dataregion/tsfile/TsFileManager.java | 6 +-
.../timeindex/FileTimeIndexCacheRecorder.java | 26 +-
.../tsfile/timeindex/PlainDeviceTimeIndex.java | 23 +-
.../dataregion/wal/buffer/WALEntry.java | 3 +
.../dataregion/wal/buffer/WALEntryType.java | 7 +-
.../dataregion/wal/io/ILogWriter.java | 13 +
.../storageengine/dataregion/wal/io/LogWriter.java | 11 +-
.../dataregion/wal/io/WALInputStream.java | 6 +-
.../storageengine/dataregion/wal/io/WALReader.java | 3 +
.../storageengine/dataregion/wal/io/WALWriter.java | 13 +-
.../file/AbstractTsFileRecoverPerformer.java | 5 +
.../file/UnsealedTsFileRecoverPerformer.java | 3 +
.../agent/SubscriptionConsumerAgent.java | 1 -
.../apache/iotdb/db/utils/ErrorHandlingUtils.java | 3 +-
.../iotdb/db/utils/constant/SqlConstant.java | 3 +
.../iotdb/db/storageengine/dataregion/TTLTest.java | 6 +-
...eCompactionWithFastPerformerValidationTest.java | 2 +-
.../cross/InsertionCrossSpaceCompactionTest.java | 4 +-
.../inner/InnerCompactionSchedulerTest.java | 8 +-
.../repair/RepairUnsortedFileCompactionTest.java | 8 +-
.../TableModelFastCompactionPerformerTest.java | 4 +-
...TableModelReadChunkCompactionPerformerTest.java | 7 +-
...TableModelReadPointCompactionPerformerTest.java | 4 +-
.../wal/compression/WALCompressionTest.java | 4 +
.../wal/recover/WALRecoverManagerTest.java | 82 ++++++
.../wal/recover/WALRepairWriterTest.java | 10 +-
.../src/test/resources/oldwal/1723544967972-1-0-0 | Bin 0 -> 237 bytes
.../datanode/src/test/resources/oldwal/_0-0-0.wal | Bin 0 -> 265150 bytes
.../src/test/resources/oldwal/_0.checkpoint | Bin 0 -> 533 bytes
.../datanode/src/test/resources/oldwal/_1-0-0.wal | Bin 0 -> 259747 bytes
.../datanode/src/test/resources/oldwal/_2-0-0.wal | Bin 0 -> 36948 bytes
.../datanode/src/test/resources/oldwal/_3-0-1.wal | Bin 0 -> 513 bytes
.../conf/iotdb-system.properties.template | 44 ++-
.../iotdb/commons/concurrent/ThreadName.java | 6 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 2 +
.../commons/exception/IllegalPathException.java | 14 +-
.../exception/IllegalPrivilegeException.java | 11 +-
.../iotdb/commons/exception/IoTDBException.java | 7 +-
.../commons/pipe/event/ProgressReportEvent.java | 5 +
.../apache/iotdb/commons/utils/StatusUtils.java | 1 +
.../src/main/thrift/iotconsensus.thrift | 9 +
.../src/main/thrift/datanode.thrift | 1 +
149 files changed, 2533 insertions(+), 455 deletions(-)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/active/ActiveLoadDirScanner.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/active/ActiveLoadPendingQueue.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/active/ActiveLoadTsFileLoader.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/{
=> limiter}/LoadTsFileRateLimiter.java (98%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/{
=> splitter}/AlignedChunkData.java (99%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/{
=> splitter}/ChunkData.java (97%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/{
=> splitter}/DeletionData.java (97%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/{
=> splitter}/NonAlignedChunkData.java (99%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/{
=> splitter}/TsFileData.java (95%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/{
=> splitter}/TsFileSplitter.java (99%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/{LoadTsfileAnalyzer.java
=> LoadTsFileAnalyzer.java} (97%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/{CompactionScheduleSummary.java
=> CompactionScheduleContext.java} (80%)
create mode 100644
iotdb-core/datanode/src/test/resources/oldwal/1723544967972-1-0-0
create mode 100644 iotdb-core/datanode/src/test/resources/oldwal/_0-0-0.wal
create mode 100644 iotdb-core/datanode/src/test/resources/oldwal/_0.checkpoint
create mode 100644 iotdb-core/datanode/src/test/resources/oldwal/_1-0-0.wal
create mode 100644 iotdb-core/datanode/src/test/resources/oldwal/_2-0-0.wal
create mode 100644 iotdb-core/datanode/src/test/resources/oldwal/_3-0-1.wal
