This is an automated email from the ASF dual-hosted git repository.
seawinde pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2d3a5b1584f [fix](fe) Keep table version monotonic after truncate
(#66255)
2d3a5b1584f is described below
commit 2d3a5b1584fc967e13b5c24bd826b6ef3b0d1fc8
Author: seawinde <[email protected]>
AuthorDate: Mon Aug 10 09:53:45 2026 +0800
[fix](fe) Keep table version monotonic after truncate (#66255)
MTMV snapshots for non-PCT base tables compare `(tableId,
tableVersion)`.
`TRUNCATE` keeps the table ID but reset the non-Cloud table version to
1,
so later writes could reuse a previously recorded snapshot and cause
AUTO
refresh to miss changed data.
**Root cause:** `InternalCatalog.truncateTableInternal()` replaced the
old
partitions and then called `OlapTable.resetVisibleVersion()`. A sequence
such
as version 3 -> TRUNCATE to 1 -> two writes to version 3 creates an ABA
collision for table-level snapshot consumers.
**Change Summary:**
| File | Change Description |
|------|--------------------|
| `InternalCatalog.java` | Increment the non-Cloud table version once
after whole-table or partition TRUNCATE; Cloud remains Meta
Service-managed |
| `OlapTable.java`, `TableAttributes.java` | Remove the TRUNCATE-only
version reset helpers |
| `TruncateTableCommandTest.java` | Verify whole-table and partition
TRUNCATE each advance the table version once |
| `test_truncate_table_mtmv.groovy` | Reproduce the non-PCT table
snapshot ABA while preserving the existing expected output |
| `truncate_version_reset.groovy` | Keep the SimpleAggCache regression
description consistent with monotonic versions |
**Design Rationale:** The table version represents table-level data
changes,
while PCT refresh separately compares partition ID and partition
version.
Reusing the existing table version avoids new persisted flags or
MTMV-specific
invalidation state. Live execution and journal replay use the same
locked path.
Cloud is skipped locally because `commit_partition` already advances and
returns the Meta Service table version.
---
.../apache/doris/datasource/InternalCatalog.java | 22 +++++++----
.../apache/doris/persist/TruncateTableInfo.java | 16 +++++++-
.../plans/commands/TruncateTableCommandTest.java | 5 +++
.../doris/persist/TruncateTableInfoTest.java | 46 ++++++++++++++++++++++
.../data/mtmv_p0/test_truncate_table_mtmv.out | 4 ++
.../suites/mtmv_p0/test_truncate_table_mtmv.groovy | 30 +++++++++++++-
.../truncate_version_reset.groovy | 32 ++-------------
7 files changed, 117 insertions(+), 38 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 793122ddb0b..aa68e03d49b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -3801,8 +3801,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
//replace
Map<Long, RecyclePartitionParam> recyclePartitionParamMap = new
HashMap<>();
+ long version = Config.isNotCloudMode() ?
olapTable.getNextVersion() : 0L;
+ long versionTimeMs = Config.isNotCloudMode() ?
System.currentTimeMillis() : 0L;
oldPartitions = truncateTableInternal(olapTable, newPartitions,
- truncateEntireTable, recyclePartitionParamMap, forceDrop);
+ truncateEntireTable, recyclePartitionParamMap, forceDrop,
version, versionTimeMs);
if (truncateEntireTable) {
Env.getCurrentEnv().getAnalysisManager().removeTableStats(olapTable.getId());
} else {
@@ -3814,7 +3816,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
TruncateTableInfo info =
new TruncateTableInfo(db.getId(), db.getFullName(),
olapTable.getId(), olapTable.getName(),
newPartitions, truncateEntireTable,
- rawTruncateSql, oldPartitions, forceDrop,
updateRecords);
+ rawTruncateSql, oldPartitions, forceDrop,
updateRecords, version, versionTimeMs);
Env.getCurrentEnv().getEditLog().logTruncateTable(info);
} catch (DdlException e) {
failedCleanCallback.run();
@@ -3830,7 +3832,8 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
private List<Partition> truncateTableInternal(OlapTable olapTable,
List<Partition> newPartitions,
- boolean isEntireTable, Map<Long, RecyclePartitionParam>
recyclePartitionParamMap, boolean isforceDrop) {
+ boolean isEntireTable, Map<Long, RecyclePartitionParam>
recyclePartitionParamMap, boolean isforceDrop,
+ long version, long versionTimeMs) {
// use new partitions to replace the old ones.
List<Partition> oldPartitions = Lists.newArrayList();
for (Partition newPartition : newPartitions) {
@@ -3860,9 +3863,13 @@ public class InternalCatalog implements
CatalogIf<Database> {
olapTable.dropPartitionForTruncate(olapTable.getDatabase().getId(),
isforceDrop, pair.getValue());
}
- // Reset table-level visibleVersion to TABLE_INIT_VERSION so it stays
consistent
- // with the newly created partitions (which also start at
PARTITION_INIT_VERSION).
- olapTable.resetVisibleVersion();
+ if (Config.isNotCloudMode() && version > 0) {
+ // Persisted values make the version transition deterministic
during journal replay.
+ olapTable.updateVisibleVersionAndTime(version, versionTimeMs);
+ } else {
+ // Preserve legacy replay and Cloud's local cache invalidation
behavior.
+ olapTable.resetVisibleVersion();
+ }
return oldPartitions;
}
@@ -3876,7 +3883,8 @@ public class InternalCatalog implements
CatalogIf<Database> {
try {
Map<Long, RecyclePartitionParam> recyclePartitionParamMap = new
HashMap<>();
truncateTableInternal(olapTable, info.getPartitions(),
info.isEntireTable(),
- recyclePartitionParamMap, isForceDrop);
+ recyclePartitionParamMap, isForceDrop,
+ info.getVersion(),
info.getVersionTimeMs());
// add tablet to inverted index
TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
index b846d1acbdc..7edffceaf0b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
@@ -60,6 +60,10 @@ public class TruncateTableInfo implements Writable {
private Map<Long, Long> updateRecords;
@SerializedName(value = "ut")
private long updateTime;
+ @SerializedName(value = "version")
+ private long version;
+ @SerializedName(value = "versionTime")
+ private long versionTimeMs;
public TruncateTableInfo() {
@@ -68,7 +72,7 @@ public class TruncateTableInfo implements Writable {
// for internal table
public TruncateTableInfo(long dbId, String db, long tblId, String table,
List<Partition> partitions,
boolean isEntireTable, String rawSql, List<Partition>
oldPartitions, boolean force,
- Map<Long, Long> updateRecords) {
+ Map<Long, Long> updateRecords, long version, long versionTimeMs) {
this.dbId = dbId;
this.db = db;
this.tblId = tblId;
@@ -81,6 +85,8 @@ public class TruncateTableInfo implements Writable {
}
this.force = force;
this.updateRecords = updateRecords;
+ this.version = version;
+ this.versionTimeMs = versionTimeMs;
}
// for external table
@@ -144,6 +150,14 @@ public class TruncateTableInfo implements Writable {
return updateTime;
}
+ public long getVersion() {
+ return version;
+ }
+
+ public long getVersionTimeMs() {
+ return versionTimeMs;
+ }
+
public static TruncateTableInfo read(DataInput in) throws IOException {
String json = Text.readString(in);
return GsonUtils.GSON.fromJson(json, TruncateTableInfo.class);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommandTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommandTest.java
index 3bc1e8a43df..53bd64e64d4 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommandTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommandTest.java
@@ -166,11 +166,15 @@ public class TruncateTableCommandTest extends
TestWithFeService {
checkShowTabletResultNum("internal.testcommand.tblcommand",
"p20210903", 4);
checkShowTabletResultNum("internal.testcommand.tblcommand",
"p20210904", 5);
+ OlapTable table =
Env.getCurrentInternalCatalog().getDbNullable("testcommand")
+ .getOlapTableOrDdlException("tblcommand");
+ long visibleVersion = table.getVisibleVersion();
String truncateStr = "truncate table internal.testcommand.tblcommand;";
NereidsParser nereidsParser = new NereidsParser();
LogicalPlan plan = nereidsParser.parseSingle(truncateStr);
Assertions.assertTrue(plan instanceof TruncateTableCommand);
Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan);
+ Assertions.assertEquals(visibleVersion + 1, table.getVisibleVersion());
checkShowTabletResultNum("internal.testcommand.tblcommand",
"p20210901", 2);
checkShowTabletResultNum("internal.testcommand.tblcommand",
"p20210902", 3);
@@ -181,6 +185,7 @@ public class TruncateTableCommandTest extends
TestWithFeService {
plan = nereidsParser.parseSingle(truncateStr);
Assertions.assertTrue(plan instanceof TruncateTableCommand);
Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan);
+ Assertions.assertEquals(visibleVersion + 2, table.getVisibleVersion());
checkShowTabletResultNum("internal.testcommand.tblcommand",
"p20210901", 2);
checkShowTabletResultNum("internal.testcommand.tblcommand",
"p20210902", 3);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/persist/TruncateTableInfoTest.java
b/fe/fe-core/src/test/java/org/apache/doris/persist/TruncateTableInfoTest.java
new file mode 100644
index 00000000000..7a5ec9a4db9
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/persist/TruncateTableInfoTest.java
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.persist;
+
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class TruncateTableInfoTest {
+ @Test
+ void testVersionSerialization() {
+ TruncateTableInfo info = new TruncateTableInfo(1L, "db", 2L, "tbl",
Lists.newArrayList(),
+ true, "TRUNCATE TABLE tbl", Lists.newArrayList(), true,
Maps.newHashMap(), 10L, 20L);
+
+ TruncateTableInfo deserialized =
GsonUtils.GSON.fromJson(info.toJson(), TruncateTableInfo.class);
+
+ Assertions.assertEquals(10L, deserialized.getVersion());
+ Assertions.assertEquals(20L, deserialized.getVersionTimeMs());
+ }
+
+ @Test
+ void testLegacyVersionDefaults() {
+ TruncateTableInfo deserialized = GsonUtils.GSON.fromJson("{}",
TruncateTableInfo.class);
+
+ Assertions.assertEquals(0L, deserialized.getVersion());
+ Assertions.assertEquals(0L, deserialized.getVersionTimeMs());
+ }
+}
diff --git a/regression-test/data/mtmv_p0/test_truncate_table_mtmv.out
b/regression-test/data/mtmv_p0/test_truncate_table_mtmv.out
index 1cbcc0cd370..baf4feba49a 100644
--- a/regression-test/data/mtmv_p0/test_truncate_table_mtmv.out
+++ b/regression-test/data/mtmv_p0/test_truncate_table_mtmv.out
@@ -8,5 +8,9 @@
2 2
3 3
+-- !non_pct_truncate --
+2 2
+3 3
+
-- !truncate_table --
diff --git a/regression-test/suites/mtmv_p0/test_truncate_table_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_truncate_table_mtmv.groovy
index 3e390d0c905..ed7f9b642aa 100644
--- a/regression-test/suites/mtmv_p0/test_truncate_table_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_truncate_table_mtmv.groovy
@@ -23,6 +23,7 @@ suite("test_truncate_table_mtmv","mtmv") {
String mvName = "${suiteName}_mv"
sql """drop table if exists `${tableName}`"""
sql """drop materialized view if exists ${mvName};"""
+ sql """drop table if exists test_truncate_table_mtmv_dim"""
sql """
CREATE TABLE ${tableName}
@@ -42,6 +43,17 @@ suite("test_truncate_table_mtmv","mtmv") {
"replication_num" = "1"
);
"""
+ sql """
+ CREATE TABLE test_truncate_table_mtmv_dim (
+ k2 TINYINT NOT NULL
+ )
+ UNIQUE KEY(k2)
+ DISTRIBUTED BY HASH(k2) BUCKETS 1
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'enable_unique_key_merge_on_write' = 'true'
+ )
+ """
sql """
CREATE MATERIALIZED VIEW ${mvName}
BUILD DEFERRED REFRESH AUTO ON MANUAL
@@ -51,12 +63,15 @@ suite("test_truncate_table_mtmv","mtmv") {
'replication_num' = '1'
)
AS
- SELECT * from ${tableName};
+ SELECT fact.k2, fact.k3
+ FROM ${tableName} fact
+ INNER JOIN test_truncate_table_mtmv_dim dim ON fact.k2 = dim.k2;
"""
sql """
insert into ${tableName} values(1,1),(2,2),(3,3);
"""
+ sql """insert into test_truncate_table_mtmv_dim values(1),(2),(3)"""
sql """
REFRESH MATERIALIZED VIEW ${mvName} AUTO
"""
@@ -73,6 +88,19 @@ suite("test_truncate_table_mtmv","mtmv") {
waitingMTMVTaskFinishedByMvName(mvName)
order_qt_truncate_partition "SELECT * FROM ${mvName}"
+ // Save a non-PCT table snapshot at version 3 while the MV does not
contain k2=2.
+ sql """delete from test_truncate_table_mtmv_dim where k2=2"""
+ sql """REFRESH MATERIALIZED VIEW ${mvName} COMPLETE"""
+ waitingMTMVTaskFinishedByMvName(mvName)
+
+ // Without monotonic table versions, reset to 1 plus two inserts collides
with version 3.
+ sql """truncate table test_truncate_table_mtmv_dim"""
+ sql """insert into test_truncate_table_mtmv_dim values(1),(2)"""
+ sql """insert into test_truncate_table_mtmv_dim values(3)"""
+ sql """REFRESH MATERIALIZED VIEW ${mvName} AUTO"""
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_non_pct_truncate "SELECT * FROM ${mvName}"
+
// truncate table
sql """
truncate table ${tableName};
diff --git
a/regression-test/suites/nereids_rules_p0/rewrite_simple_agg_to_constant/truncate_version_reset.groovy
b/regression-test/suites/nereids_rules_p0/rewrite_simple_agg_to_constant/truncate_version_reset.groovy
index caef303ff5b..0d86ca64cd0 100644
---
a/regression-test/suites/nereids_rules_p0/rewrite_simple_agg_to_constant/truncate_version_reset.groovy
+++
b/regression-test/suites/nereids_rules_p0/rewrite_simple_agg_to_constant/truncate_version_reset.groovy
@@ -16,25 +16,8 @@
// under the License.
/**
- * Regression test for: TRUNCATE TABLE must reset
TableAttributes.visibleVersion.
- *
- * Bug: before the fix, truncateTableInternal() replaced partition data but did
- * not call olapTable.resetVisibleVersion(). As a result:
- * - Partition.visibleVersion was reset to PARTITION_INIT_VERSION (1).
- * - TableAttributes.visibleVersion kept its old, higher value.
- * - TableAttributes.visibleVersionTime was never updated.
- *
- * Consequence for RewriteSimpleAggToConstantRule / SimpleAggCacheMgr:
- * The cache entry was keyed by versionTime. Because versionTime did not
- * change at truncate time, the *caller* saw the same versionTime as the
- * stale cached entry → cache HIT → the rule returned the pre-truncate
- * count/min/max instead of the correct post-truncate values.
- *
- * The fix adds olapTable.resetVisibleVersion() inside truncateTableInternal(),
- * which bumps both visibleVersion (back to TABLE_INIT_VERSION = 1) and
- * visibleVersionTime (to System.currentTimeMillis()). The new versionTime
- * differs from the cached entry's versionTime → cache MISS → the rule
- * correctly falls back to BE execution and returns the right result.
+ * TRUNCATE changes visible rows, so it must advance the table version
metadata.
+ * SimpleAggCacheMgr uses that metadata to invalidate cached aggregate values.
*/
suite("truncate_version_reset") {
sql "SET enable_nereids_planner=true"
@@ -86,23 +69,14 @@ suite("truncate_version_reset") {
// Confirm the cached count is correct before truncate.
order_qt_count_before_truncate "SELECT count(*) FROM tbl;"
- // -----------------------------------------------------------------------
- // TRUNCATE the table.
- // After the fix, resetVisibleVersion() is called inside
- // truncateTableInternal(), which updates visibleVersionTime.
- // The cache entry's versionTime no longer matches → cache is invalidated.
- // -----------------------------------------------------------------------
+ // Truncate must invalidate the cached aggregate value.
sql "TRUNCATE TABLE tbl;"
// count(*) must return 0.
// Without the fix, the stale cache entry (count = 5) would be returned.
order_qt_count_after_truncate "SELECT count(*) FROM tbl;"
- // -----------------------------------------------------------------------
// Insert new rows after truncate, then verify count(*) reflects them.
- // This also validates that the version counter is correctly reset so
- // subsequent transactions start from the right next-version.
- // -----------------------------------------------------------------------
sql "INSERT INTO tbl VALUES (10, 100), (20, 200);"
// After insert the count must be 2.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
