This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new ff698529a2e branch-3.1: [improve](statistics)Improve drop expired
stats function. #54290 (#54310)
ff698529a2e is described below
commit ff698529a2e05436c8dc5668c0594d5c0adf0bcf
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Aug 5 14:56:30 2025 +0800
branch-3.1: [improve](statistics)Improve drop expired stats function.
#54290 (#54310)
Cherry-picked from #54290
Co-authored-by: James <[email protected]>
---
.../apache/doris/statistics/StatisticsCleaner.java | 13 ++++--
.../doris/statistics/StatisticsCleanerTest.java | 53 ++++++++++++++++++++++
2 files changed, 62 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
index a1c5fc4edc5..e5ab2851ea3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
@@ -141,7 +141,7 @@ public class StatisticsCleaner extends MasterDaemon {
analysisManager.removeTableStats(id);
Env.getCurrentEnv().getEditLog().logDeleteTableStats(new
TableStatsDeletionLog(id));
} catch (Exception e) {
- LOG.info(e);
+ LOG.info("Fail to remove table stats for table {}", id, e);
}
}
}
@@ -257,13 +257,18 @@ public class StatisticsCleaner extends MasterDaemon {
}
}
- private long findExpiredStats(OlapTable statsTbl, ExpiredStats
expiredStats,
+ protected long findExpiredStats(OlapTable statsTbl, ExpiredStats
expiredStats,
long offset, boolean isTableColumnStats) {
long pos = offset;
- while (pos < statsTbl.getRowCount() && !expiredStats.isFull()) {
+ while (!expiredStats.isFull()) {
List<ResultRow> rows = StatisticsRepository.fetchStatsFullName(
StatisticConstants.FETCH_LIMIT, pos, isTableColumnStats);
pos += StatisticConstants.FETCH_LIMIT;
+ if (rows.isEmpty()) {
+ LOG.info("Stats table {} has no more rows to fetch.",
statsTbl.getName());
+ break;
+ }
+ LOG.info("Process {} rows in stats table {}", rows.size(),
statsTbl.getName());
for (ResultRow r : rows) {
try {
StatsId statsId = new StatsId(r);
@@ -324,7 +329,7 @@ public class StatisticsCleaner extends MasterDaemon {
return pos;
}
- private static class ExpiredStats {
+ protected static class ExpiredStats {
Set<Long> expiredCatalog = new HashSet<>();
Set<Long> expiredDatabase = new HashSet<>();
Set<Long> expiredTable = new HashSet<>();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCleanerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCleanerTest.java
new file mode 100644
index 00000000000..32524b0c132
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCleanerTest.java
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.common.Config;
+import org.apache.doris.statistics.StatisticsCleaner.ExpiredStats;
+
+import com.google.common.collect.Lists;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+class StatisticsCleanerTest {
+
+ @Test
+ void testFindExpiredStats() {
+ StatisticsCleaner cleaner = new StatisticsCleaner();
+ ExpiredStats stat = new ExpiredStats();
+ OlapTable olapTable = new OlapTable();
+ for (int i = 0; i <= Config.max_allowed_in_element_num_of_delete; i++)
{
+ stat.expiredCatalog.add((long) i);
+ }
+ long expiredStats = cleaner.findExpiredStats(null, stat, 1, true);
+ Assertions.assertEquals(expiredStats, 1);
+
+ try (MockedStatic<StatisticsRepository> mockedRep =
Mockito.mockStatic(StatisticsRepository.class)) {
+ mockedRep.when(() -> StatisticsRepository.fetchStatsFullName(
+ ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong(),
ArgumentMatchers.anyBoolean()))
+ .thenReturn(Lists.newArrayList());
+ stat.expiredCatalog.clear();
+ expiredStats = cleaner.findExpiredStats(olapTable, stat, 0, true);
+ Assertions.assertEquals(expiredStats,
StatisticConstants.FETCH_LIMIT);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
