This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 722d5a6c558 branch-4.1 [fix](fe) Fix simple aggregate cache after
partition recycle (#63530)
722d5a6c558 is described below
commit 722d5a6c558a8c0ab2f0fafd057f4923a3ac8145
Author: minghong <[email protected]>
AuthorDate: Fri May 22 23:41:27 2026 +0800
branch-4.1 [fix](fe) Fix simple aggregate cache after partition recycle
(#63530)
### What problem does this PR solve?
pick #63175
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../org/apache/doris/catalog/TableAttributes.java | 4 +-
.../apache/doris/datasource/InternalCatalog.java | 11 ++-
.../doris/nereids/stats/SimpleAggCacheMgr.java | 12 +++
.../auto_partition_recycle_count_cache.groovy | 91 ++++++++++++++++++++++
4 files changed, 114 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableAttributes.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableAttributes.java
index b581ffd4713..847cdf7265b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableAttributes.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableAttributes.java
@@ -33,9 +33,9 @@ public class TableAttributes {
@SerializedName(value = "constraints")
private final Map<String, Constraint> constraintsMap = new HashMap<>();
@SerializedName(value = "visibleVersion")
- private long visibleVersion;
+ private volatile long visibleVersion;
@SerializedName(value = "visibleVersionTime")
- private long visibleVersionTime;
+ private volatile long visibleVersionTime;
public TableAttributes() {
this.visibleVersion = TABLE_INIT_VERSION;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 4e7764662d1..96230b04665 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -127,6 +127,7 @@ import org.apache.doris.info.PartitionNamesInfo;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.stats.SimpleAggCacheMgr;
import
org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand.IdType;
import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo;
import org.apache.doris.persist.AlterDatabasePropertyInfo;
@@ -1975,6 +1976,9 @@ public class InternalCatalog implements
CatalogIf<Database> {
olapTable.updateVisibleVersionAndTime(version, versionTime);
}
}
+ if (partition != null && !isTempPartition) {
+
SimpleAggCacheMgr.internalInstance().invalidateTable(olapTable.getId());
+ }
// Here, we only wait for the EventProcessor to finish processing the
event,
// but regardless of the success or failure of the result,
@@ -2006,15 +2010,18 @@ public class InternalCatalog implements
CatalogIf<Database> {
OlapTable olapTable = (OlapTable)
db.getTableOrMetaException(info.getTableId(), TableType.OLAP);
olapTable.writeLock();
try {
+ Partition partition = null;
if (info.isTempPartition()) {
olapTable.dropTempPartition(info.getPartitionName(), true);
} else {
- Partition partition = olapTable.dropPartition(info.getDbId(),
info.getPartitionName(),
- info.isForceDrop());
+ partition = olapTable.dropPartition(info.getDbId(),
info.getPartitionName(), info.isForceDrop());
if (!info.isForceDrop() && partition != null &&
info.getRecycleTime() != 0) {
Env.getCurrentRecycleBin().setRecycleTimeByIdForReplay(partition.getId(),
info.getRecycleTime());
}
}
+ if (partition != null && !info.isTempPartition()) {
+
SimpleAggCacheMgr.internalInstance().invalidateTable(olapTable.getId());
+ }
olapTable.updateVisibleVersionAndTime(info.getVersion(),
info.getVersionTime());
// Replay set new partition loaded flag to true for auto analyze.
TableStatsMeta stats =
Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(olapTable.getId());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/SimpleAggCacheMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/SimpleAggCacheMgr.java
index cc9cf81d275..1b67a7cd35d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/SimpleAggCacheMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/SimpleAggCacheMgr.java
@@ -317,6 +317,18 @@ public class SimpleAggCacheMgr {
cache.synchronous().invalidate(key);
}
+ /**
+ * Evict all cached simple aggregate values for a table.
+ */
+ public void invalidateTable(long tableId) {
+ if (rowCountCache != null) {
+ rowCountCache.synchronous().invalidate(tableId);
+ }
+ if (cache != null) {
+ cache.synchronous().asMap().keySet().removeIf(key ->
key.getTableId() == tableId);
+ }
+ }
+
/**
* Get the cached row count for a table.
*
diff --git
a/regression-test/suites/nereids_rules_p0/rewrite_simple_agg_to_constant/auto_partition_recycle_count_cache.groovy
b/regression-test/suites/nereids_rules_p0/rewrite_simple_agg_to_constant/auto_partition_recycle_count_cache.groovy
new file mode 100644
index 00000000000..5519f0b5162
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/rewrite_simple_agg_to_constant/auto_partition_recycle_count_cache.groovy
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("auto_partition_recycle_count_cache", "nonConcurrent") {
+ def oldCheckInterval =
getFeConfig("dynamic_partition_check_interval_seconds")
+
+ try {
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+ sql "DROP TABLE IF EXISTS auto_partition_recycle_count_cache FORCE"
+ sql """
+ CREATE TABLE auto_partition_recycle_count_cache (
+ k0 DATETIME(6) NOT NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(k0)
+ AUTO PARTITION BY RANGE(date_trunc(k0, 'day')) ()
+ DISTRIBUTED BY HASH(k0) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "partition.retention_count" = "3"
+ )
+ """
+
+ sql """
+ INSERT INTO auto_partition_recycle_count_cache VALUES
+ ('2020-01-01 00:00:00'),
+ ('2020-01-02 00:00:00'),
+ ('2020-01-03 00:00:00'),
+ ('2020-01-04 00:00:00'),
+ ('2020-01-05 00:00:00'),
+ ('2020-01-06 00:00:00')
+ """
+ sql "sync"
+
+ def partitionCount = sql "SHOW PARTITIONS FROM
auto_partition_recycle_count_cache"
+ assertEquals(6, partitionCount.size())
+
+ sql "SELECT count(*) FROM auto_partition_recycle_count_cache"
+ def cacheReady = false
+ for (int i = 0; i < 30; i++) {
+ def explainResult = sql "EXPLAIN SELECT count(*) FROM
auto_partition_recycle_count_cache"
+ if (explainResult.toString().contains("constant exprs")) {
+ cacheReady = true
+ break
+ }
+ sleep(1000)
+ }
+ if (!cacheReady) {
+ if (isCloudMode()) {
+ logger.info("SimpleAggCacheMgr did not warm up in cloud mode,
skip")
+ return
+ }
+ assertTrue(false, "SimpleAggCacheMgr cache did not warm up within
30 seconds")
+ }
+
+ def countBeforeRecycle = sql "SELECT count(*) FROM
auto_partition_recycle_count_cache"
+ assertEquals(6L, countBeforeRecycle[0][0] as long)
+
+ setFeConfig("dynamic_partition_check_interval_seconds", 1)
+ def recycled = false
+ for (int i = 0; i < 30; i++) {
+ partitionCount = sql "SHOW PARTITIONS FROM
auto_partition_recycle_count_cache"
+ if (partitionCount.size() == 3) {
+ recycled = true
+ break
+ }
+ sleep(1000)
+ }
+ assertTrue(recycled, "auto partition retention did not recycle
partitions within 30 seconds")
+
+ def countAfterRecycle = sql "SELECT count(*) FROM
auto_partition_recycle_count_cache"
+ assertEquals(3L, countAfterRecycle[0][0] as long)
+ } finally {
+ setFeConfig("dynamic_partition_check_interval_seconds",
oldCheckInterval)
+ sql "DROP TABLE IF EXISTS auto_partition_recycle_count_cache FORCE"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
