This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 138da30a7e7 [fix](fe) Return unknown stats for system tables (#62913)
138da30a7e7 is described below
commit 138da30a7e7272f1b2adca80345b8048a292319a
Author: yujun <[email protected]>
AuthorDate: Wed May 6 14:56:29 2026 +0800
[fix](fe) Return unknown stats for system tables (#62913)
### What problem does this PR solve?
Related PR: introduced by #41790
Problem Summary:
We met a case where manually dropping column stats tablet files caused
many internal queries against `__internal_schema.column_statistics`,
including queries whose target stats IDs belong to the statistics table
itself. The visible symptom is a self-amplifying internal-query storm
such as:
```sql
SELECT * FROM __internal_schema.column_statistics
WHERE id IN (... column_statistics own column stats ids ...)
```
The problematic call chain is:
```text
ColumnStatisticsCacheLoader.doLoad
-> StatisticsRepository.loadColStats
-> StatisticsUtil.execStatisticQuery
-> StmtExecutor.executeInternalQuery
-> NereidsPlanner.optimize
-> InitJoinOrder
-> StatsCalculator.disableJoinReorderIfStatsInvalid
-> StatsCalculator.checkNdvValidation
-> StatisticsCache.OlapTableStatistics.getColumnStatistics
```
When the internal stats-loading SQL scans
`__internal_schema.column_statistics`, join-reorder's pre-validation
path runs before `computeOlapScan()` derives UNKNOWN stats for system
tables. That pre-validation can request column statistics for
`column_statistics` itself. If the stats load fails, the async stats
cache does not retain a useful value for the failed key, so repeated
planning can reload the same system-table stats and amplify the internal
query volume.
This is separate from the audit `State=OK` display issue: those audit
rows can look successful even when the internal stats query failed. The
bug fixed here is the recursive system-table stats lookup during
planning.
This PR returns UNKNOWN from `StatisticsCache.OlapTableStatistics`
column and partition-column accessors for system tables. That keeps the
system-table behavior consistent with `computeOlapScan()` and prevents
future callers through this accessor from accidentally loading
system-table stats before the normal scan-statistics guard.
The fix intentionally does not skip all internal queries. Internal jobs
that scan normal user tables, such as import, MV refresh, or internal
insert tasks, can still use normal stats validation and optimization.
---
.../apache/doris/statistics/StatisticsCache.java | 6 +++
.../doris/statistics/StatisticsCacheTest.java | 46 ++++++++++++++++++++++
2 files changed, 52 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index e8a5c519660..6ba538b81a3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -401,6 +401,9 @@ public class StatisticsCache {
if (ctx != null && ctx.getState().isPlanWithUnKnownColumnStats()) {
return ColumnStatistic.UNKNOWN;
}
+ if (StatisticConstants.isSystemTable(olapTable)) {
+ return ColumnStatistic.UNKNOWN;
+ }
return doGetColumnStatistics(
catalogId, schemaId, tableId, selectIndexId, colName, ctx
);
@@ -411,6 +414,9 @@ public class StatisticsCache {
if (ctx != null && ctx.getState().isPlanWithUnKnownColumnStats()) {
return PartitionColumnStatistic.UNKNOWN;
}
+ if (StatisticConstants.isSystemTable(olapTable)) {
+ return PartitionColumnStatistic.UNKNOWN;
+ }
return doGetPartitionColumnStatistics(
catalogId, schemaId, tableId, selectIndexId, partName,
colName
);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCacheTest.java
index cb1dffbe106..712be0ef0b6 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCacheTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCacheTest.java
@@ -17,6 +17,11 @@
package org.apache.doris.statistics;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.common.FeConstants;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.nereids.trees.plans.algebra.OlapScan;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.UtFrameUtils;
@@ -24,6 +29,8 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.MockedConstruction;
+import org.mockito.Mockito;
public class StatisticsCacheTest {
@@ -87,4 +94,43 @@ public class StatisticsCacheTest {
ConnectContext.get().getState().setPlanWithUnKnownColumnStats(prevFlag);
}
}
+
+ @Test
+ public void testOlapTableStatisticsSkipSystemTable() {
+ try (MockedConstruction<ColumnStatisticsCacheLoader> columnLoader =
Mockito.mockConstruction(
+ ColumnStatisticsCacheLoader.class);
+ MockedConstruction<PartitionColumnStatisticCacheLoader>
partitionLoader = Mockito.mockConstruction(
+ PartitionColumnStatisticCacheLoader.class)) {
+ StatisticsCache cache = new StatisticsCache();
+ StatisticsCache.OlapTableStatistics olapTableStats =
+
cache.getOlapTableStats(mockSystemOlapScan(FeConstants.INTERNAL_DB_NAME));
+
+ Assertions.assertEquals(ColumnStatistic.UNKNOWN,
+ olapTableStats.getColumnStatistics("col",
ConnectContext.get()));
+ Assertions.assertEquals(PartitionColumnStatistic.UNKNOWN,
+ olapTableStats.getPartitionColumnStatistics("p1", "col",
ConnectContext.get()));
+
+ Mockito.verifyNoInteractions(columnLoader.constructed().get(0));
+ Mockito.verifyNoInteractions(partitionLoader.constructed().get(0));
+ }
+ }
+
+ private OlapScan mockSystemOlapScan(String dbName) {
+ CatalogIf catalog = Mockito.mock(CatalogIf.class);
+ Mockito.when(catalog.getId()).thenReturn(1L);
+ DatabaseIf database = Mockito.mock(DatabaseIf.class);
+ Mockito.when(database.getId()).thenReturn(2L);
+ Mockito.when(database.getCatalog()).thenReturn(catalog);
+
+ OlapTable table = Mockito.mock(OlapTable.class);
+ Mockito.when(table.getQualifiedDbName()).thenReturn(dbName);
+ Mockito.when(table.getDatabase()).thenReturn(database);
+ Mockito.when(table.getId()).thenReturn(3L);
+ Mockito.when(table.getBaseIndexId()).thenReturn(4L);
+
+ OlapScan scan = Mockito.mock(OlapScan.class);
+ Mockito.when(scan.getTable()).thenReturn(table);
+ Mockito.when(scan.getSelectedIndexId()).thenReturn(4L);
+ return scan;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
