This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new f6802a4053b branch-3.1: [improve](statistic)Reduce call of olap table
getVisibleVersion for cloud version. #53903 (#54205)
f6802a4053b is described below
commit f6802a4053bd7d4c0c252b3a561bb927ede59e5a
Author: James <[email protected]>
AuthorDate: Mon Aug 4 10:00:03 2025 +0800
branch-3.1: [improve](statistic)Reduce call of olap table getVisibleVersion
for cloud version. #53903 (#54205)
backport: #53903
---
.../doris/statistics/StatisticsAutoCollector.java | 85 ++++++++++++----------
.../doris/statistics/StatisticsJobAppender.java | 58 ++++++++-------
.../doris/statistics/util/StatisticsUtil.java | 19 +++--
.../statistics/StatisticsAutoCollectorTest.java | 66 +++++++++++------
.../statistics/StatisticsJobAppenderTest.java | 78 +++++++++-----------
.../doris/statistics/util/StatisticsUtilTest.java | 34 ++++++---
6 files changed, 187 insertions(+), 153 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 512e807c620..8cd96bc4957 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -28,7 +28,6 @@ import org.apache.doris.common.Pair;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.persist.TableStatsDeletionLog;
-import org.apache.doris.rpc.RpcException;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
@@ -140,23 +139,32 @@ public class StatisticsAutoCollector extends MasterDaemon
{
}
}
- protected void processOneJob(TableIf table, Set<Pair<String, String>>
columns,
- JobPriority priority) throws DdlException {
- appendAllColumns(table, columns);
- AnalysisMethod analysisMethod = table.getDataSize(true) >=
StatisticsUtil.getHugeTableLowerBoundSizeInBytes()
+ protected void processOneJob(TableIf table, Set<Pair<String, String>>
columns, JobPriority priority) {
+ AnalysisMethod analysisMethod =
(StatisticsUtil.getHugeTableLowerBoundSizeInBytes() == 0
+ || table.getDataSize(true) >=
StatisticsUtil.getHugeTableLowerBoundSizeInBytes())
? AnalysisMethod.SAMPLE : AnalysisMethod.FULL;
if (StatisticsUtil.enablePartitionAnalyze() &&
table.isPartitionedTable()) {
analysisMethod = AnalysisMethod.FULL;
}
boolean isSampleAnalyze = analysisMethod.equals(AnalysisMethod.SAMPLE);
OlapTable olapTable = table instanceof OlapTable ? (OlapTable) table :
null;
+ AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
+ TableStatsMeta tableStatsStatus =
manager.findTableStatsStatus(table.getId());
+ long rowCount = table.getRowCount();
+ if (!readyToSample(table, rowCount, manager, tableStatsStatus,
isSampleAnalyze)) {
+ return;
+ }
+ appendAllColumns(table, columns);
+ long olapTableVersion = StatisticsUtil.getOlapTableVersion(olapTable);
columns = columns.stream()
- .filter(c -> StatisticsUtil.needAnalyzeColumn(table, c) ||
StatisticsUtil.isLongTimeColumn(table, c))
+ .filter(c -> StatisticsUtil.needAnalyzeColumn(table, c)
+ || StatisticsUtil.isLongTimeColumn(table, c,
olapTableVersion))
.filter(c -> olapTable == null ||
StatisticsUtil.canCollectColumn(
olapTable.getIndexMetaByIndexId(olapTable.getIndexIdByName(c.first)).getColumnByName(c.second),
table, isSampleAnalyze,
olapTable.getIndexIdByName(c.first)))
.collect(Collectors.toSet());
- AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns,
priority, analysisMethod);
+ AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns,
priority, analysisMethod,
+ rowCount, tableStatsStatus, olapTableVersion);
if (analyzeJob == null) {
return;
}
@@ -172,8 +180,34 @@ public class StatisticsAutoCollector extends MasterDaemon {
}
}
+ protected boolean readyToSample(TableIf table, long rowCount,
AnalysisManager manager,
+ TableStatsMeta tableStatsStatus, boolean isSample) {
+ if (!isSample) {
+ return true;
+ }
+ OlapTable olapTable = table instanceof OlapTable ? (OlapTable) table :
null;
+ if (olapTable != null
+ && olapTable.getRowCountForIndex(olapTable.getBaseIndexId(),
true) == TableIf.UNKNOWN_ROW_COUNT) {
+ LOG.info("Table {} row count is not fully reported, skip auto
analyzing it.", olapTable.getName());
+ return false;
+ }
+ // We don't auto analyze empty table to avoid all 0 stats.
+ // Because all 0 is more dangerous than unknown stats when row count
report is delayed.
+ if (rowCount <= 0) {
+ LOG.info("Table {} is empty, remove its old stats and skip auto
analyze it.", table.getName());
+ // Remove the table's old stats if exists.
+ if (tableStatsStatus != null &&
!tableStatsStatus.isColumnsStatsEmpty()) {
+ manager.removeTableStats(table.getId());
+ Env.getCurrentEnv().getEditLog().logDeleteTableStats(new
TableStatsDeletionLog(table.getId()));
+ manager.dropStats(table, null);
+ }
+ return false;
+ }
+ return true;
+ }
+
// If partition changed (partition first loaded, partition dropped and so
on), need re-analyze all columns.
- protected void appendAllColumns(TableIf table, Set<Pair<String, String>>
columns) throws DdlException {
+ private void appendAllColumns(TableIf table, Set<Pair<String, String>>
columns) {
if (!(table instanceof OlapTable)) {
return;
}
@@ -198,30 +232,9 @@ public class StatisticsAutoCollector extends MasterDaemon {
&& ((HMSExternalTable)
tableIf).getDlaType().equals(HMSExternalTable.DLAType.HIVE);
}
- protected AnalysisInfo createAnalyzeJobForTbl(
- TableIf table, Set<Pair<String, String>> jobColumns, JobPriority
priority, AnalysisMethod analysisMethod) {
- AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
- TableStatsMeta tableStatsStatus =
manager.findTableStatsStatus(table.getId());
- if (table instanceof OlapTable &&
analysisMethod.equals(AnalysisMethod.SAMPLE)) {
- OlapTable ot = (OlapTable) table;
- if (ot.getRowCountForIndex(ot.getBaseIndexId(), true) ==
TableIf.UNKNOWN_ROW_COUNT) {
- LOG.info("Table {} row count is not fully reported, skip auto
analyzing this time.", ot.getName());
- return null;
- }
- }
- // We don't auto analyze empty table to avoid all 0 stats.
- // Because all 0 is more dangerous than unknown stats when row count
report is delayed.
- long rowCount = table.getRowCount();
- if (rowCount <= 0) {
- LOG.info("Table {} is empty, remove its old stats and skip auto
analyze it.", table.getName());
- // Remove the table's old stats if exists.
- if (tableStatsStatus != null &&
!tableStatsStatus.isColumnsStatsEmpty()) {
- manager.removeTableStats(table.getId());
- Env.getCurrentEnv().getEditLog().logDeleteTableStats(new
TableStatsDeletionLog(table.getId()));
- manager.dropStats(table, null);
- }
- return null;
- }
+ protected AnalysisInfo createAnalyzeJobForTbl(TableIf table,
Set<Pair<String, String>> jobColumns,
+ JobPriority priority, AnalysisMethod analysisMethod, long
rowCount, TableStatsMeta tableStatsStatus,
+ long version) {
if (jobColumns == null || jobColumns.isEmpty()) {
return null;
}
@@ -230,14 +243,6 @@ public class StatisticsAutoCollector extends MasterDaemon {
for (Pair<String, String> pair : jobColumns) {
stringJoiner.add(pair.toString());
}
- long version = 0;
- try {
- if (table instanceof OlapTable) {
- version = ((OlapTable) table).getVisibleVersion();
- }
- } catch (RpcException e) {
- LOG.warn("table {}, in cloud getVisibleVersion exception",
table.getName(), e);
- }
return new AnalysisInfoBuilder()
.setJobId(Env.getCurrentEnv().getNextId())
.setCatalogId(table.getDatabase().getCatalog().getId())
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
index 7597c25633f..3ddd360fb5e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.Config;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.datasource.InternalCatalog;
@@ -31,6 +32,7 @@ import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -144,10 +146,10 @@ public class StatisticsJobAppender extends MasterDaemon {
}
}
- protected void appendToLowJobs(Map<TableName, Set<Pair<String, String>>>
lowPriorityJobs,
+ protected int appendToLowJobs(Map<TableName, Set<Pair<String, String>>>
lowPriorityJobs,
Map<TableName, Set<Pair<String, String>>>
veryLowPriorityJobs) {
if (System.currentTimeMillis() - lastRoundFinishTime <
lowJobIntervalMs) {
- return;
+ return 0;
}
InternalCatalog catalog = Env.getCurrentInternalCatalog();
List<Long> sortedDbs =
catalog.getDbIds().stream().sorted().collect(Collectors.toList());
@@ -173,35 +175,34 @@ public class StatisticsJobAppender extends MasterDaemon {
Set<String> columns = t.getSchemaAllIndexes(false).stream()
.filter(c ->
!StatisticsUtil.isUnsupportedType(c.getType()))
.map(Column::getName).collect(Collectors.toSet());
- Set<Pair<String, String>> columnIndexPairs =
t.getColumnIndexPairs(columns)
- .stream().filter(p ->
StatisticsUtil.needAnalyzeColumn(t, p))
- .collect(Collectors.toSet());
TableName tableName = new
TableName(t.getDatabase().getCatalog().getName(),
t.getDatabase().getFullName(), t.getName());
- // Append to low job map first.
- if (!columnIndexPairs.isEmpty()) {
- boolean appended = doAppend(lowPriorityJobs,
columnIndexPairs, tableName);
- // If low job map is full, stop this iteration.
- if (!appended) {
- LOG.debug("Low Priority job map is full.");
- return;
- }
- } else {
- // Append to very low job map.
- columnIndexPairs = t.getColumnIndexPairs(columns)
- .stream().filter(p ->
StatisticsUtil.isLongTimeColumn(t, p))
- .collect(Collectors.toSet());
- if (!columnIndexPairs.isEmpty()) {
- boolean appended = doAppend(veryLowPriorityJobs,
columnIndexPairs, tableName);
- // If very low job map is full, simply ignore it and
go to the next table.
- if (!appended) {
+ boolean appended = false;
+ long version = Config.isCloudMode() ? 0 :
StatisticsUtil.getOlapTableVersion((OlapTable) t);
+ for (Pair<String, String> p : t.getColumnIndexPairs(columns)) {
+ // Append to low job map first.
+ if (StatisticsUtil.needAnalyzeColumn(t, p)) {
+ // If low job map is full, stop this iteration.
+ if (!doAppend(lowPriorityJobs, p, tableName)) {
+ LOG.debug("Low Priority job map is full.");
+ return processed;
+ }
+ appended = true;
+ } else if (StatisticsUtil.isLongTimeColumn(t, p, version))
{
+ // If very low job map is full, simply ignore it and
go to the next column.
+ if (!doAppend(veryLowPriorityJobs, p, tableName)) {
LOG.debug("Very low Priority job map is full.");
+ } else {
+ appended = true;
}
}
}
currentTableId = t.getId();
- if (++processed >= TABLE_BATCH_SIZE) {
- return;
+ if (appended) {
+ processed++;
+ }
+ if (processed >= TABLE_BATCH_SIZE) {
+ return processed;
}
}
}
@@ -212,6 +213,7 @@ public class StatisticsJobAppender extends MasterDaemon {
currentDbId = 0;
currentTableId = 0;
lastRoundFinishTime = System.currentTimeMillis();
+ return processed;
}
protected List<Table> sortTables(List<Table> tables) {
@@ -223,16 +225,18 @@ public class StatisticsJobAppender extends MasterDaemon {
@VisibleForTesting
public boolean doAppend(Map<TableName, Set<Pair<String, String>>> jobMap,
- Set<Pair<String, String>> columnIndexPairs,
+ Pair<String, String> columnIndexPair,
TableName tableName) {
synchronized (jobMap) {
if (!jobMap.containsKey(tableName) && jobMap.size() >=
JOB_MAP_SIZE) {
return false;
}
if (jobMap.containsKey(tableName)) {
- jobMap.get(tableName).addAll(columnIndexPairs);
+ jobMap.get(tableName).add(columnIndexPair);
} else {
- jobMap.put(tableName, columnIndexPairs);
+ Set<Pair<String, String>> columnSet = Sets.newHashSet();
+ columnSet.add(columnIndexPair);
+ jobMap.put(tableName, columnSet);
}
}
return true;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 8989cfd9b87..7cbbfc43591 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -1180,7 +1180,7 @@ public class StatisticsUtil {
}
// This function return true means the column hasn't been analyzed for
longer than the configured time.
- public static boolean isLongTimeColumn(TableIf table, Pair<String, String>
column) {
+ public static boolean isLongTimeColumn(TableIf table, Pair<String, String>
column, long version) {
if (column == null) {
return false;
}
@@ -1213,15 +1213,20 @@ public class StatisticsUtil {
}
// For olap table, if the table visible version and row count doesn't
change since last analyze,
// we don't need to analyze it because its data is not changed.
- OlapTable olapTable = (OlapTable) table;
- long version = 0;
+ return version != columnStats.tableVersion
+ || table.getRowCount() != columnStats.rowCount;
+ }
+
+ public static long getOlapTableVersion(OlapTable olapTable) {
+ if (olapTable == null) {
+ return 0;
+ }
try {
- version = ((OlapTable) table).getVisibleVersion();
+ return olapTable.getVisibleVersion();
} catch (RpcException e) {
- LOG.warn("in cloud getVisibleVersion exception", e);
+ LOG.warn("table {}, in cloud getVisibleVersion exception",
olapTable.getName(), e);
+ return 0;
}
- return version != columnStats.tableVersion
- || olapTable.getRowCount() != columnStats.rowCount;
}
public static boolean canCollect() {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
index 095a0df5138..625fd904809 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
@@ -19,12 +19,15 @@ package org.apache.doris.statistics;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalDatabase;
import org.apache.doris.datasource.hive.HMSExternalTable;
@@ -35,10 +38,12 @@ import org.apache.doris.datasource.jdbc.JdbcExternalTable;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import mockit.Mock;
import mockit.MockUp;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
import java.util.ArrayList;
import java.util.HashSet;
@@ -155,30 +160,45 @@ public class StatisticsAutoCollectorTest {
@Test
public void testCreateAnalyzeJobForTbl() {
StatisticsAutoCollector collector = new StatisticsAutoCollector();
- OlapTable table = new OlapTable();
- new MockUp<OlapTable>() {
- @Mock
- public long getDataSize(boolean singleReplica) {
- return 100;
- }
+ OlapTable table = Mockito.mock(OlapTable.class);
+ Database db = Mockito.mock(Database.class);
+ InternalCatalog catalog = Mockito.mock(InternalCatalog.class);
+ Mockito.when(table.getDatabase()).thenReturn(db);
+ Mockito.when(db.getCatalog()).thenReturn(catalog);
+ Mockito.when(db.getId()).thenReturn(100L);
+ Mockito.when(catalog.getId()).thenReturn(10L);
+
+
+ Assertions.assertNull(collector.createAnalyzeJobForTbl(table, null,
null, AnalysisMethod.SAMPLE, 100, null, 10));
+
+ Set<Pair<String, String>> jobColumns = Sets.newHashSet();
+ jobColumns.add(Pair.of("a", "b"));
+ jobColumns.add(Pair.of("c", "d"));
+ AnalysisInfo analyzeJobForTbl =
collector.createAnalyzeJobForTbl(table, jobColumns, JobPriority.HIGH,
AnalysisMethod.SAMPLE, 100,
+ null, 10);
+ Assertions.assertEquals("[a:b,c:d]", analyzeJobForTbl.colName);
+ Assertions.assertEquals(JobPriority.HIGH, analyzeJobForTbl.priority);
+ Assertions.assertEquals(AnalysisMethod.SAMPLE,
analyzeJobForTbl.analysisMethod);
+ Assertions.assertEquals(100, analyzeJobForTbl.rowCount);
+ Assertions.assertEquals(10, analyzeJobForTbl.tableVersion);
+ }
- @Mock
- public long getRowCountForIndex(long indexId, boolean strict) {
- return -1;
- }
+ @Test
+ public void testReadyToSample() {
+ StatisticsAutoCollector collector = new StatisticsAutoCollector();
+ OlapTable table = Mockito.mock(OlapTable.class);
+ Mockito.when(table.getName()).thenReturn("table");
+ Mockito.when(table.getRowCountForIndex(Mockito.anyLong(),
Mockito.anyBoolean())).thenReturn(TableIf.UNKNOWN_ROW_COUNT);
+ // not sample
+ Assertions.assertTrue(collector.readyToSample(table, 100, null, null,
false));
+ // not fully reported.
+ Assertions.assertFalse(collector.readyToSample(table, 100, null, null,
true));
+
+ Mockito.when(table.getRowCountForIndex(Mockito.anyLong(),
Mockito.anyBoolean())).thenReturn(100L);
+ // Row count is 0
+ Assertions.assertFalse(collector.readyToSample(table, 0, null, null,
true));
+ // ready to sample
+ Assertions.assertTrue(collector.readyToSample(table, 100, null, null,
true));
- @Mock
- public boolean isPartitionedTable() {
- return false;
- }
- };
- Assertions.assertNull(collector.createAnalyzeJobForTbl(table, null,
null, AnalysisMethod.SAMPLE));
- new MockUp<OlapTable>() {
- @Mock
- public long getRowCountForIndex(long indexId, boolean strict) {
- return 100;
- }
- };
- Assertions.assertNull(collector.createAnalyzeJobForTbl(table, null,
null, AnalysisMethod.SAMPLE));
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java
index 94c50c91ee3..aa7f3fc06d4 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java
@@ -34,7 +34,6 @@ import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
import mockit.Mock;
import mockit.MockUp;
import org.junit.jupiter.api.Assertions;
@@ -192,30 +191,29 @@ public class StatisticsJobAppenderTest {
StatisticsJobAppender appender = new StatisticsJobAppender();
appender.appendToLowJobs(testLowMap, testVeryLowMap);
Assertions.assertEquals(100, testLowMap.size());
+ Assertions.assertEquals(0, testVeryLowMap.size());
testLowMap.clear();
appender.appendToLowJobs(testLowMap, testVeryLowMap);
Assertions.assertEquals(40, testLowMap.size());
-
- for (int i = 0; i < StatisticsJobAppender.JOB_MAP_SIZE; i++) {
- Database db = new Database(id++, "testDb" + i);
- testCatalog.unprotectCreateDb(db);
- Column column1 = new Column("placeholder", PrimitiveType.INT);
- List<Column> schema = new ArrayList<>();
- schema.add(column1);
- OlapTable table1 = new OlapTable(id++, "testTable" + id + "_1",
schema, null, null, null);
- OlapTable table2 = new OlapTable(id++, "testTable" + id + "_1",
schema, null, null, null);
- db.createTableWithLock(table1, true, false);
- db.createTableWithLock(table2, true, false);
- }
-
+ Assertions.assertEquals(0, testVeryLowMap.size());
testLowMap.clear();
- appender.setLastRoundFinishTime(0);
+ // Less than 1 minutes since last iteration.
appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ Assertions.assertEquals(0, testLowMap.size());
+ Assertions.assertEquals(0, testVeryLowMap.size());
+
+ testLowMap.clear();
appender.setLastRoundFinishTime(0);
- appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ int processed = appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ Assertions.assertEquals(100, testLowMap.size());
+ Assertions.assertEquals(0, testVeryLowMap.size());
+ Assertions.assertEquals(100, processed);
appender.setLastRoundFinishTime(0);
- appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ processed = appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ Assertions.assertEquals(100, testLowMap.size());
+ Assertions.assertEquals(0, testVeryLowMap.size());
Assertions.assertEquals(StatisticsJobAppender.JOB_MAP_SIZE,
testLowMap.size());
+ Assertions.assertEquals(0, processed);
}
@Test
@@ -260,7 +258,7 @@ public class StatisticsJobAppenderTest {
}
@Mock
- public boolean isLongTimeColumn(TableIf table, Pair<String,
String> column) {
+ public boolean isLongTimeColumn(TableIf table, Pair<String,
String> column, long version) {
return true;
}
};
@@ -268,33 +266,28 @@ public class StatisticsJobAppenderTest {
Map<TableName, Set<Pair<String, String>>> testLowMap = new HashMap<>();
Map<TableName, Set<Pair<String, String>>> testVeryLowMap = new
HashMap<>();
StatisticsJobAppender appender = new StatisticsJobAppender();
- appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ int processed = appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ Assertions.assertEquals(0, testLowMap.size());
Assertions.assertEquals(100, testVeryLowMap.size());
+ Assertions.assertEquals(100, processed);
testVeryLowMap.clear();
- appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ processed = appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ Assertions.assertEquals(0, testLowMap.size());
Assertions.assertEquals(40, testVeryLowMap.size());
-
- for (int i = 0; i < StatisticsJobAppender.JOB_MAP_SIZE; i++) {
- Database db = new Database(id++, "testDb" + i);
- testCatalog.unprotectCreateDb(db);
- Column column1 = new Column("placeholder", PrimitiveType.INT);
- List<Column> schema = new ArrayList<>();
- schema.add(column1);
- OlapTable table1 = new OlapTable(id++, "testTable" + id + "_1",
schema, null, null, null);
- OlapTable table2 = new OlapTable(id++, "testTable" + id + "_1",
schema, null, null, null);
- db.createTableWithLock(table1, true, false);
- db.createTableWithLock(table2, true, false);
- }
+ Assertions.assertEquals(40, processed);
testLowMap.clear();
appender.setLastRoundFinishTime(0);
- appender.appendToLowJobs(testLowMap, testVeryLowMap);
- appender.setLastRoundFinishTime(0);
- appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ processed = appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ Assertions.assertEquals(0, testLowMap.size());
+ Assertions.assertEquals(100, testVeryLowMap.size());
+ Assertions.assertEquals(100, processed);
+
appender.setLastRoundFinishTime(0);
- appender.appendToLowJobs(testLowMap, testVeryLowMap);
+ processed = appender.appendToLowJobs(testLowMap, testVeryLowMap);
Assertions.assertEquals(0, testLowMap.size());
- Assertions.assertEquals(StatisticsJobAppender.JOB_MAP_SIZE,
testVeryLowMap.size());
+ Assertions.assertEquals(100, testVeryLowMap.size());
+ Assertions.assertEquals(0, processed);
}
@Test
@@ -348,23 +341,19 @@ public class StatisticsJobAppenderTest {
@Test
public void testDoAppend() {
Map<TableName, Set<Pair<String, String>>> jobMap = Maps.newHashMap();
- Set<Pair<String, String>> columnIndexPairs1 = Sets.newHashSet();
- Set<Pair<String, String>> columnIndexPairs2 = Sets.newHashSet();
TableName tableName1 = new TableName("catalog1", "db1", "table1");
TableName tableName2 = new TableName("catalog2", "db2", "table2");
Pair<String, String> pair1 = Pair.of("index1", "col1");
- columnIndexPairs1.add(pair1);
StatisticsJobAppender appender = new StatisticsJobAppender();
- Assertions.assertTrue(appender.doAppend(jobMap, columnIndexPairs1,
tableName1));
+ Assertions.assertTrue(appender.doAppend(jobMap, pair1, tableName1));
Assertions.assertEquals(1, jobMap.size());
Assertions.assertTrue(jobMap.containsKey(tableName1));
Assertions.assertEquals(1, jobMap.get(tableName1).size());
Assertions.assertTrue(jobMap.get(tableName1).contains(pair1));
Pair<String, String> pair2 = Pair.of("index2", "col2");
- columnIndexPairs1.add(pair2);
- Assertions.assertTrue(appender.doAppend(jobMap, columnIndexPairs1,
tableName1));
+ Assertions.assertTrue(appender.doAppend(jobMap, pair2, tableName1));
Assertions.assertEquals(1, jobMap.size());
Assertions.assertTrue(jobMap.containsKey(tableName1));
Assertions.assertEquals(2, jobMap.get(tableName1).size());
@@ -372,8 +361,7 @@ public class StatisticsJobAppenderTest {
Assertions.assertTrue(jobMap.get(tableName1).contains(pair2));
Pair<String, String> pair3 = Pair.of("index3", "col3");
- columnIndexPairs2.add(pair3);
- Assertions.assertTrue(appender.doAppend(jobMap, columnIndexPairs2,
tableName2));
+ Assertions.assertTrue(appender.doAppend(jobMap, pair3, tableName2));
Assertions.assertEquals(2, jobMap.size());
Assertions.assertTrue(jobMap.containsKey(tableName2));
Assertions.assertEquals(1, jobMap.get(tableName2).size());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
index 8b641e01ebd..3ad3b6c977b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
@@ -43,6 +43,7 @@ import org.apache.doris.datasource.jdbc.JdbcExternalCatalog;
import org.apache.doris.datasource.jdbc.JdbcExternalDatabase;
import org.apache.doris.datasource.jdbc.JdbcExternalTable;
import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.rpc.RpcException;
import org.apache.doris.statistics.AnalysisManager;
import org.apache.doris.statistics.ColStatsMeta;
import org.apache.doris.statistics.ResultRow;
@@ -56,6 +57,7 @@ import mockit.MockUp;
import org.apache.iceberg.CatalogProperties;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
import java.nio.charset.StandardCharsets;
import java.time.LocalTime;
@@ -381,7 +383,7 @@ class StatisticsUtilTest {
OlapTable table = new OlapTable(200, "testTable", schema, null, null,
null);
// Test column is null
- Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, null));
+ Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, null,
0));
// Test table auto analyze is disabled.
new MockUp<OlapTable>() {
@@ -390,7 +392,7 @@ class StatisticsUtilTest {
return false;
}
};
- Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName())));
+ Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName()), 0));
new MockUp<OlapTable>() {
@Mock
public boolean autoAnalyzeEnabled() {
@@ -410,7 +412,7 @@ class StatisticsUtilTest {
props.put(CatalogProperties.WAREHOUSE_LOCATION, "s3://tmp");
IcebergExternalCatalog catalog = new IcebergHadoopExternalCatalog(0,
"iceberg_ctl", "", props, "");
IcebergExternalTable icebergTable = new IcebergExternalTable(0, "",
"", catalog, icebergDatabase);
- Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(icebergTable,
Pair.of("index", column.getName())));
+ Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(icebergTable,
Pair.of("index", column.getName()), 0));
// Test table stats meta is null.
new MockUp<AnalysisManager>() {
@@ -419,7 +421,7 @@ class StatisticsUtilTest {
return null;
}
};
- Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName())));
+ Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName()), 0));
// Test column stats meta is null
TableStatsMeta tableMeta = new TableStatsMeta();
@@ -435,7 +437,7 @@ class StatisticsUtilTest {
return null;
}
};
- Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName())));
+ Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName()), 0));
new MockUp<TableStatsMeta>() {
@Mock
public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
@@ -445,16 +447,16 @@ class StatisticsUtilTest {
// Test table stats is user injected
tableMeta.userInjected = true;
- Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName())));
+ Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName()), 0));
tableMeta.userInjected = false;
// Test Config.auto_analyze_interval_seconds == 0
Config.auto_analyze_interval_seconds = 0;
- Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName())));
+ Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName()), 0));
// Test column analyzed within the time interval
Config.auto_analyze_interval_seconds = 86400;
- Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName())));
+ Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName()), 0));
// Test column hasn't analyzed for longer than time interval, but
version and row count doesn't change
new MockUp<TableStatsMeta>() {
@@ -481,7 +483,7 @@ class StatisticsUtilTest {
}
};
Config.auto_analyze_interval_seconds = 1;
- Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName())));
+ Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName()), 10));
// Test column hasn't analyzed for longer than time interval, and
version change
new MockUp<OlapTable>() {
@@ -495,7 +497,7 @@ class StatisticsUtilTest {
return 100;
}
};
- Assertions.assertTrue(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName())));
+ Assertions.assertTrue(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName()), 11));
// Test column hasn't analyzed for longer than time interval, and row
count change
new MockUp<OlapTable>() {
@@ -509,7 +511,7 @@ class StatisticsUtilTest {
return 101;
}
};
- Assertions.assertTrue(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName())));
+ Assertions.assertTrue(StatisticsUtil.isLongTimeColumn(table,
Pair.of("index", column.getName()), 10));
}
@Test
@@ -565,4 +567,14 @@ class StatisticsUtilTest {
Assertions.assertTrue(StatisticsUtil.canCollectColumn(column, table,
true, 1));
}
+
+ @Test
+ public void testGetOlapTableVersion() throws RpcException {
+ Assertions.assertEquals(0, StatisticsUtil.getOlapTableVersion(null));
+ OlapTable ot = Mockito.mock(OlapTable.class);
+ Mockito.when(ot.getVisibleVersion()).thenReturn(100L);
+ Assertions.assertEquals(100, StatisticsUtil.getOlapTableVersion(ot));
+ Mockito.when(ot.getVisibleVersion()).thenThrow(new RpcException("",
""));
+ Assertions.assertEquals(0, StatisticsUtil.getOlapTableVersion(ot));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
