This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8b52996cd9a [test](nereids) Replace flaky null stats case with unit
test (#65445)
8b52996cd9a is described below
commit 8b52996cd9a74dba08423044e9ef3b154fd66b3b
Author: shuke <[email protected]>
AuthorDate: Mon Jul 13 17:40:40 2026 +0800
[test](nereids) Replace flaky null stats case with unit test (#65445)
### What problem does this PR solve?
Related PR: #62265
Problem Summary:
`test_scale_num_nulls` checks that table-level `numNulls=3` is scaled to
`1` when partition pruning selects 4 of 12 rows. The regression case
executes `EXPLAIN MEMO PLAN` immediately after insert and analyze, but
Cloud P0 reports the physical partition row count asynchronously. The
planner can therefore observe a transient selected-partition row count
of zero, while waiting for the real row count can take up to two
minutes.
This change:
- adds deterministic FE unit coverage that invokes
`StatsCalculator.computeOlapScan()` with a three-partition OLAP scan;
- fixes the table row count at 12, selected-partition row count at 4,
and cached table-level column statistics at `ndv=1`, `min=max=2`, and
`numNulls=3`;
- verifies the final scan statistics have `ndv=1`, `min=max=2`,
`count=4`, and scaled `numNulls=1`;
- removes the timing-dependent Groovy regression case.
The unit test exercises the production partition-pruning statistics path
using fixed in-memory inputs. It has no cluster, physical row-count
reporting, polling, sleep, network, or storage dependency. Production
behavior is unchanged.
---
.../doris/nereids/stats/StatsCalculatorTest.java | 75 ++++++++++++++++++++++
.../suites/statistics/test_scale_num_nulls.groovy | 60 -----------------
2 files changed, 75 insertions(+), 60 deletions(-)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
index 8bfab4c4ec1..ae6d93e17fc 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
@@ -17,8 +17,11 @@
package org.apache.doris.nereids.stats;
+import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
@@ -54,9 +57,11 @@ import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanConstructor;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.statistics.AnalysisManager;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
import org.apache.doris.statistics.Statistics;
+import org.apache.doris.statistics.StatisticsCache;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -64,6 +69,7 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
import org.mockito.Mockito;
import java.lang.reflect.Field;
@@ -206,6 +212,75 @@ public class StatsCalculatorTest {
Assertions.assertNotNull(stats.columnStatistics().get(slot1));
}
+ @Test
+ public void testComputeOlapScanScalesNumNullsForSelectedPartitions() {
+ ConnectContext previousContext = ConnectContext.get();
+ ConnectContext connectContext = new ConnectContext();
+ connectContext.setThreadLocalInfo();
+
+ Env env = Mockito.mock(Env.class);
+ AnalysisManager analysisManager = Mockito.mock(AnalysisManager.class);
+ StatisticsCache statisticsCache = Mockito.mock(StatisticsCache.class);
+ StatisticsCache.OlapTableStatistics olapTableStatistics
+ = Mockito.mock(StatisticsCache.OlapTableStatistics.class);
+ OlapTable table = Mockito.mock(OlapTable.class);
+ LogicalOlapScan scan = Mockito.mock(LogicalOlapScan.class);
+ Partition selectedPartition = Mockito.mock(Partition.class);
+
+ long selectedPartitionId = 1L;
+ long baseIndexId = 10L;
+ Column column = new Column("val", PrimitiveType.INT);
+ SlotReference slot = new SlotReference(new ExprId(1), "val",
IntegerType.INSTANCE, true,
+ ImmutableList.of("test", "tbl"), table, column, table, column);
+ ColumnStatistic tableStatistic = new ColumnStatisticBuilder(12)
+ .setNdv(1)
+ .setMinValue(2)
+ .setMaxValue(2)
+ .setMinExpr(new IntLiteral(2))
+ .setMaxExpr(new IntLiteral(2))
+ .setNumNulls(3)
+ .build();
+
+ Mockito.when(env.getAnalysisManager()).thenReturn(analysisManager);
+ Mockito.when(env.getStatisticsCache()).thenReturn(statisticsCache);
+
Mockito.when(statisticsCache.getOlapTableStats(scan)).thenReturn(olapTableStatistics);
+ Mockito.when(olapTableStatistics.getColumnStatistics("val",
connectContext)).thenReturn(tableStatistic);
+ Mockito.when(scan.getTable()).thenReturn(table);
+ Mockito.when(scan.getSelectedIndexId()).thenReturn(baseIndexId);
+
Mockito.when(scan.getSelectedPartitionIds()).thenReturn(ImmutableList.of(selectedPartitionId));
+ Mockito.when(scan.getOutput()).thenReturn(ImmutableList.of(slot));
+ Mockito.when(scan.getVirtualColumns()).thenReturn(ImmutableList.of());
+ Mockito.when(table.getBaseIndexId()).thenReturn(baseIndexId);
+ Mockito.when(table.getRowCountForIndex(baseIndexId,
true)).thenReturn(12L);
+ Mockito.when(table.getRowCountForPartitionIndex(selectedPartitionId,
baseIndexId, true)).thenReturn(4L);
+ Mockito.when(table.getPartitionNum()).thenReturn(3);
+
Mockito.when(table.getPartition(selectedPartitionId)).thenReturn(selectedPartition);
+ Mockito.when(table.getQualifiedDbName()).thenReturn("test");
+ Mockito.when(selectedPartition.getName()).thenReturn("p1");
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+
+ Statistics statistics = new StatsCalculator((CascadesContext)
null).computeOlapScan(scan);
+ ColumnStatistic result = statistics.findColumnStatistics(slot);
+
+ Assertions.assertEquals(4, statistics.getRowCount(), 0.001);
+ Assertions.assertNotNull(result);
+ Assertions.assertEquals(1, result.ndv, 0.001);
+ Assertions.assertEquals(2, result.minValue, 0.001);
+ Assertions.assertEquals(2, result.maxValue, 0.001);
+ Assertions.assertEquals(4, result.count, 0.001);
+ Assertions.assertEquals(1, result.numNulls, 0.001);
+ Assertions.assertEquals("2", result.minExpr.getStringValue());
+ Assertions.assertEquals("2", result.maxExpr.getStringValue());
+ } finally {
+ ConnectContext.remove();
+ if (previousContext != null) {
+ previousContext.setThreadLocalInfo();
+ }
+ }
+ }
+
@Test
public void testLimit() {
List<String> qualifier = ImmutableList.of("test", "t");
diff --git a/regression-test/suites/statistics/test_scale_num_nulls.groovy
b/regression-test/suites/statistics/test_scale_num_nulls.groovy
deleted file mode 100644
index 043b072ca62..00000000000
--- a/regression-test/suites/statistics/test_scale_num_nulls.groovy
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_scale_num_nulls") {
- // For an OlapTable, when only a subset of partitions is selected,
- // the num_nulls value in column statistics needs to be scaled
proportionally.
- sql """
- drop table if exists ptable;
- CREATE TABLE `ptable` (
- `id` int NULL,
- `val` int NULL,
- `d` date NULL
- ) ENGINE=OLAP
- DUPLICATE KEY(`id`)
- PARTITION BY RANGE(`d`)
- (PARTITION p201701 VALUES [('2017-01-01'), ('2017-02-01')),
- PARTITION p201702 VALUES [('2017-02-01'), ('2017-03-01')),
- PARTITION p201703 VALUES [('2017-03-01'), ('2017-04-01')))
- DISTRIBUTED BY HASH(`id`) BUCKETS 16
- PROPERTIES (
- "replication_allocation" = "tag.location.default: 1",
- "min_load_replica_num" = "-1",
- "is_being_synced" = "false",
- "storage_medium" = "hdd",
- "storage_format" = "V2",
- "inverted_index_storage_format" = "V3",
- "light_schema_change" = "true",
- "disable_auto_compaction" = "false",
- "group_commit_interval_ms" = "10000",
- "group_commit_data_bytes" = "134217728"
- );
-
- insert into ptable values
- (1, null, '2017-01-01'), (11,2, '2017-01-01'), (111,2, '2017-01-01'),
(1111,2, '2017-01-01'),
- (2, null, '2017-02-01'), (22,2, '2017-02-01'), (222,2, '2017-02-01'),
(2222,2, '2017-02-01'),
- (3, null, '2017-03-01'), (33,2, '2017-03-01'), (333,2, '2017-03-01'),
(333,2, '2017-03-01');
-
- analyze table ptable with sync;
- """
- def colStats = sql "show column stats ptable";
- def memo = sql "explain memo plan select * from ptable where
d='2017-01-01'"
- // check numNulls=1.0000 for column val, which is scaled from 3 to 1
according to the partition pruning result.
- assertTrue(memo.toString().contains("val#1 -> ndv=1.0000,
min=2.000000(2), max=2.000000(2), count=4.0000, numNulls=1.0000"),
- "numNulls should be 1.0000, but is " + memo.toString() + "\n
column stats: \n" + colStats.toString());
-}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
