This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 8faf26ea577 branch-4.2: [fix](fe) Do not scale numNulls of partition
statistics again #69636 (#70055)
8faf26ea577 is described below
commit 8faf26ea577787e5a56402ab41b680769b22a68a
Author: morrySnow <[email protected]>
AuthorDate: Wed Nov 4 08:16:35 2026 +0800
branch-4.2: [fix](fe) Do not scale numNulls of partition statistics again
#69636 (#70055)
picked from master #69636
Co-authored-by: minghong <[email protected]>
---
.../doris/nereids/stats/StatsCalculator.java | 90 +++++++------
.../doris/nereids/stats/StatsCalculatorTest.java | 145 +++++++++++++++++----
2 files changed, 174 insertions(+), 61 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index b1a8588c019..e497f6ba6c2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -616,10 +616,21 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
&&
connectContext.getSessionVariable().enablePartitionAnalyze;
for (SlotReference slot : visibleOutputSlots) {
ColumnStatistic cache;
+ // numNulls of the table level statistics covers the whole
table, so it has to be scaled
+ // to the selected partitions. numNulls of the partition level
statistics is already
+ // scoped to them and must not be scaled again.
+ boolean scaleNumNulls;
if (enablePartitionStatics) {
- cache = getColumnStatistic(olapTableStats, slot.getName(),
selectedPartitionNames);
+ cache = getPartitionColumnStatistic(olapTableStats,
slot.getName(), selectedPartitionNames);
+ // A dump stores table-level statistics, which still
require scaling.
+ scaleNumNulls = cache == null || isPlayNereidsDump;
+ if (cache == null) {
+ // partition level statistics are unavailable, fall
back to table level statistics
+ cache =
olapTableStats.getColumnStatistics(slot.getName(), connectContext);
+ }
} else {
cache = olapTableStats.getColumnStatistics(slot.getName(),
connectContext);
+ scaleNumNulls = true;
}
if (slot.getOriginalColumn().isPresent()) {
cache = updateMinMaxForPartitionKey(olapTable,
selectedPartitionNames, slot, cache);
@@ -627,9 +638,10 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
ColumnStatisticBuilder colStatsBuilder = new
ColumnStatisticBuilder(cache,
selectedPartitionsRowCount);
colStatsBuilder.normalizeAvgSizeByte(slot.getDataType());
- //scale null_num
- double scale = tableRowCount == 0 ? 1 :
selectedPartitionsRowCount / tableRowCount;
- colStatsBuilder.setNumNulls(colStatsBuilder.getNumNulls() *
scale);
+ if (scaleNumNulls) {
+ double scale = tableRowCount == 0 ? 1 :
selectedPartitionsRowCount / tableRowCount;
+ colStatsBuilder.setNumNulls(colStatsBuilder.getNumNulls()
* scale);
+ }
builder.putColumnStatistics(slot, colStatsBuilder.build());
}
checkIfUnknownStatsUsedAsKey(builder);
@@ -1256,10 +1268,13 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
}
/**
- * - partition pruned: try to get partition col stats, if failed, then
fall back to table level col stats
- * - no partition pruned: get table level col stats
+ * Merge the column statistics of the given partitions.
+ *
+ * <p>The returned statistic is scoped to the given partitions, i.e. its
count and numNulls only
+ * cover them. Returns null if the statistics of any partition is unknown,
so that the caller can
+ * decide whether to fall back to the table level column statistics.
*/
- private ColumnStatistic getColumnStatistic(
+ private ColumnStatistic getPartitionColumnStatistic(
OlapTableStatistics olapTableStatistics, String colName,
List<String> partitionNames) {
if (connectContext != null &&
connectContext.getState().isPlanWithUnKnownColumnStats()) {
return ColumnStatistic.UNKNOWN;
@@ -1271,40 +1286,37 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
} else {
return ColumnStatistic.UNKNOWN;
}
- } else {
- if (!partitionNames.isEmpty()) {
- PartitionColumnStatisticBuilder builder = new
PartitionColumnStatisticBuilder();
- boolean hasUnknown = false;
- // check if there is any unknown stats to avoid unnecessary
partition column stats merge.
- List<PartitionColumnStatistic> pColStatsLists = new
ArrayList<>(partitionNames.size());
- for (String partitionName : partitionNames) {
- PartitionColumnStatistic pcolStats
- =
olapTableStatistics.getPartitionColumnStatistics(partitionName, colName,
connectContext);
- if (pcolStats.isUnKnown) {
- hasUnknown = true;
- break;
- } else {
- pColStatsLists.add(pcolStats);
- }
- }
- if (!hasUnknown) {
- boolean isFirst = true;
- // try to merge partition column stats
- for (PartitionColumnStatistic pcolStats : pColStatsLists) {
- if (isFirst) {
- builder = new
PartitionColumnStatisticBuilder(pcolStats);
- isFirst = false;
- } else {
- builder.merge(pcolStats);
- }
- }
- return builder.toColumnStatistics();
- }
+ }
+ if (partitionNames.isEmpty()) {
+ return null;
+ }
+ PartitionColumnStatisticBuilder builder = new
PartitionColumnStatisticBuilder();
+ boolean hasUnknown = false;
+ // check if there is any unknown stats to avoid unnecessary partition
column stats merge.
+ List<PartitionColumnStatistic> pColStatsLists = new
ArrayList<>(partitionNames.size());
+ for (String partitionName : partitionNames) {
+ PartitionColumnStatistic pcolStats
+ =
olapTableStatistics.getPartitionColumnStatistics(partitionName, colName,
connectContext);
+ if (pcolStats.isUnKnown) {
+ hasUnknown = true;
+ break;
+ }
+ pColStatsLists.add(pcolStats);
+ }
+ if (hasUnknown) {
+ return null;
+ }
+ boolean isFirst = true;
+ // try to merge partition column stats
+ for (PartitionColumnStatistic pcolStats : pColStatsLists) {
+ if (isFirst) {
+ builder = new PartitionColumnStatisticBuilder(pcolStats);
+ isFirst = false;
+ } else {
+ builder.merge(pcolStats);
}
-
- // if any partition-col-stats is unknown, fall back to table level
col stats
- return olapTableStatistics.getColumnStatistics(colName,
connectContext);
}
+ return builder.toColumnStatistics();
}
/**
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
index 357d2514fb8..afa66c48159 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
@@ -59,9 +59,12 @@ import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.AnalysisManager;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
+import org.apache.doris.statistics.PartitionColumnStatistic;
+import org.apache.doris.statistics.PartitionColumnStatisticBuilder;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.statistics.StatisticsCache;
import org.apache.doris.statistics.TableStatsMeta;
+import org.apache.doris.statistics.util.Hll128;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -216,9 +219,98 @@ public class StatsCalculatorTest {
@Test
public void testComputeOlapScanScalesNumNullsForSelectedPartitions() {
+ // table level statistics: the whole table has 12 rows and 3 nulls,
the selected partition has 4 rows
+ try (MockPartitionPrunedScan fixture = mockPartitionPrunedScan(false,
newTableStatistic(1), null)) {
+ Statistics statistics = fixture.computeOlapScan();
+ ColumnStatistic result =
statistics.findColumnStatistics(fixture.slot);
+
+ Assertions.assertEquals(4, statistics.getRowCount(), 0.001);
+ Assertions.assertNotNull(result);
+ Assertions.assertEquals(1, result.ndv, 0.001);
+ Assertions.assertEquals(2, result.minValue, 0.001);
+ Assertions.assertEquals(2, result.maxValue, 0.001);
+ Assertions.assertEquals(4, result.count, 0.001);
+ Assertions.assertEquals(1, result.numNulls, 0.001);
+ Assertions.assertEquals("2", result.minExpr.getStringValue());
+ Assertions.assertEquals("2", result.maxExpr.getStringValue());
+ }
+ }
+
+ @Test
+ public void testComputeOlapScanDoesNotScaleNumNullsOfPartitionStatistics()
{
+ // partition level statistics of the selected partition: 4 rows and 1
of them is null
+ PartitionColumnStatistic partitionStatistic = new
PartitionColumnStatisticBuilder(4)
+ .setNdv(newHll128(2))
+ .setNumNulls(1)
+ .setMinValue(2)
+ .setMaxValue(2)
+ .setMinExpr(new IntLiteral(2))
+ .setMaxExpr(new IntLiteral(2))
+ .build();
+ try (MockPartitionPrunedScan fixture = mockPartitionPrunedScan(true,
+ newTableStatistic(2), partitionStatistic)) {
+ Statistics statistics = fixture.computeOlapScan();
+ ColumnStatistic result =
statistics.findColumnStatistics(fixture.slot);
+
+ Assertions.assertEquals(4, statistics.getRowCount(), 0.001);
+ Assertions.assertNotNull(result);
+ // ndv comes from the partition level statistics, the table level
ones have ndv=2
+ Assertions.assertEquals(1, result.ndv, 0.001);
+ Assertions.assertEquals(4, result.count, 0.001);
+ // numNulls of the partition level statistics already covers the
selected partition only,
+ // it must not be scaled by selectedPartitionsRowCount /
tableRowCount (1 * 4 / 12) again
+ Assertions.assertEquals(1, result.numNulls, 0.001);
+ }
+ }
+
+ @Test
+ public void
testComputeOlapScanFallsBackToTableStatisticsWhenPartitionStatisticIsUnknown() {
+ try (MockPartitionPrunedScan fixture = mockPartitionPrunedScan(true,
+ newTableStatistic(2), PartitionColumnStatistic.UNKNOWN)) {
+ Statistics statistics = fixture.computeOlapScan();
+ ColumnStatistic result =
statistics.findColumnStatistics(fixture.slot);
+
+ Assertions.assertEquals(4, statistics.getRowCount(), 0.001);
+ Assertions.assertNotNull(result);
+ // ndv comes from the table level statistics, i.e. it fell back to
them
+ Assertions.assertEquals(2, result.ndv, 0.001);
+ Assertions.assertEquals(4, result.count, 0.001);
+ // the table level statistic covers the whole table, so numNulls
is scaled: 3 * 4 / 12
+ Assertions.assertEquals(1, result.numNulls, 0.001);
+ }
+ }
+
+ private static ColumnStatistic newTableStatistic(double ndv) {
+ return new ColumnStatisticBuilder(12)
+ .setNdv(ndv)
+ .setMinValue(2)
+ .setMaxValue(2)
+ .setMinExpr(new IntLiteral(2))
+ .setMaxExpr(new IntLiteral(2))
+ .setNumNulls(3)
+ .build();
+ }
+
+ private static Hll128 newHll128(long value) {
+ Hll128 hll128 = new Hll128();
+ hll128.update(value);
+ return hll128;
+ }
+
+ /**
+ * Mock an OLAP scan over a table with 3 partitions and 12 rows, whose
partition pruning selects
+ * exactly one partition with 4 rows.
+ *
+ * @param enablePartitionStatics value of the session variable
enable_partition_analyze
+ * @param tableStatistic statistics of the whole table
+ * @param partitionStatistic statistics of the selected partition, null if
not stubbed
+ */
+ private static MockPartitionPrunedScan mockPartitionPrunedScan(boolean
enablePartitionStatics,
+ ColumnStatistic tableStatistic, PartitionColumnStatistic
partitionStatistic) {
ConnectContext previousContext = ConnectContext.get();
ConnectContext connectContext = new ConnectContext();
connectContext.setThreadLocalInfo();
+ connectContext.getSessionVariable().enablePartitionAnalyze =
enablePartitionStatics;
Env env = Mockito.mock(Env.class);
AnalysisManager analysisManager = Mockito.mock(AnalysisManager.class);
@@ -234,19 +326,15 @@ public class StatsCalculatorTest {
Column column = new Column("val", PrimitiveType.INT);
SlotReference slot = new SlotReference(new ExprId(1), "val",
IntegerType.INSTANCE, true,
ImmutableList.of("test", "tbl"), table, column, table, column);
- ColumnStatistic tableStatistic = new ColumnStatisticBuilder(12)
- .setNdv(1)
- .setMinValue(2)
- .setMaxValue(2)
- .setMinExpr(new IntLiteral(2))
- .setMaxExpr(new IntLiteral(2))
- .setNumNulls(3)
- .build();
Mockito.when(env.getAnalysisManager()).thenReturn(analysisManager);
Mockito.when(env.getStatisticsCache()).thenReturn(statisticsCache);
Mockito.when(statisticsCache.getOlapTableStats(scan)).thenReturn(olapTableStatistics);
Mockito.when(olapTableStatistics.getColumnStatistics("val",
connectContext)).thenReturn(tableStatistic);
+ if (partitionStatistic != null) {
+
Mockito.when(olapTableStatistics.getPartitionColumnStatistics("p1", "val",
connectContext))
+ .thenReturn(partitionStatistic);
+ }
Mockito.when(scan.getTable()).thenReturn(table);
Mockito.when(scan.getSelectedIndexId()).thenReturn(baseIndexId);
Mockito.when(scan.getSelectedPartitionIds()).thenReturn(ImmutableList.of(selectedPartitionId));
@@ -260,22 +348,35 @@ public class StatsCalculatorTest {
Mockito.when(table.getQualifiedDbName()).thenReturn("test");
Mockito.when(selectedPartition.getName()).thenReturn("p1");
- try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
- mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class);
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ return new MockPartitionPrunedScan(previousContext, scan, slot,
mockedEnv);
+ }
- Statistics statistics = new StatsCalculator((CascadesContext)
null).computeOlapScan(scan);
- ColumnStatistic result = statistics.findColumnStatistics(slot);
+ /**
+ * Mocks of {@link #mockPartitionPrunedScan}, they have to be closed after
computing the statistics.
+ */
+ private static final class MockPartitionPrunedScan implements
AutoCloseable {
+ private final ConnectContext previousContext;
+ private final LogicalOlapScan scan;
+ private final SlotReference slot;
+ private final MockedStatic<Env> mockedEnv;
+
+ private MockPartitionPrunedScan(ConnectContext previousContext,
LogicalOlapScan scan, SlotReference slot,
+ MockedStatic<Env> mockedEnv) {
+ this.previousContext = previousContext;
+ this.scan = scan;
+ this.slot = slot;
+ this.mockedEnv = mockedEnv;
+ }
- Assertions.assertEquals(4, statistics.getRowCount(), 0.001);
- Assertions.assertNotNull(result);
- Assertions.assertEquals(1, result.ndv, 0.001);
- Assertions.assertEquals(2, result.minValue, 0.001);
- Assertions.assertEquals(2, result.maxValue, 0.001);
- Assertions.assertEquals(4, result.count, 0.001);
- Assertions.assertEquals(1, result.numNulls, 0.001);
- Assertions.assertEquals("2", result.minExpr.getStringValue());
- Assertions.assertEquals("2", result.maxExpr.getStringValue());
- } finally {
+ private Statistics computeOlapScan() {
+ return new StatsCalculator((CascadesContext)
null).computeOlapScan(scan);
+ }
+
+ @Override
+ public void close() {
+ mockedEnv.close();
ConnectContext.remove();
if (previousContext != null) {
previousContext.setThreadLocalInfo();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
