This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 713ae3e5802 [improvement](statistics) Add config for the threshold of
column count for auto analyze #27713 (#27723)
713ae3e5802 is described below
commit 713ae3e5802d6b6c8a338b220edd058d0e3b4aef
Author: Jibing-Li <[email protected]>
AuthorDate: Wed Nov 29 03:01:25 2023 +0800
[improvement](statistics) Add config for the threshold of column count for
auto analyze #27713 (#27723)
---
.../java/org/apache/doris/qe/SessionVariable.java | 9 +++++++
.../doris/statistics/StatisticConstants.java | 2 ++
.../doris/statistics/StatisticsAutoCollector.java | 6 ++++-
.../doris/statistics/util/StatisticsUtil.java | 10 +++++++
.../statistics/StatisticsAutoCollectorTest.java | 31 ++++++++++++++++++++++
5 files changed, 57 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 01086001588..3f42414ddaf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -411,6 +411,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_AUTO_ANALYZE = "enable_auto_analyze";
+ public static final String AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD =
"auto_analyze_table_width_threshold";
+
public static final String FASTER_FLOAT_CONVERT = "faster_float_convert";
public static final String ENABLE_DECIMAL256 = "enable_decimal256";
@@ -1194,6 +1196,13 @@ public class SessionVariable implements Serializable,
Writable {
flag = VariableMgr.GLOBAL)
public boolean enableAutoAnalyze = true;
+ @VariableMgr.VarAttr(name = AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD,
+ description = {"参与自动收集的最大表宽度，列数多于这个参数的表不参与自动收集",
+ "Maximum table width to enable auto analyze, "
+ + "table with more columns than this value will not be
auto analyzed."},
+ flag = VariableMgr.GLOBAL)
+ public int autoAnalyzeTableWidthThreshold = 70;
+
@VariableMgr.VarAttr(name = AUTO_ANALYZE_START_TIME, needForward = true,
checker = "checkAnalyzeTimeFormat",
description = {"该参数定义自动ANALYZE例程的开始时间",
"This parameter defines the start time for the automatic
ANALYZE routine."},
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
index ffb80074eee..3e2b9c8bc28 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
@@ -98,6 +98,8 @@ public class StatisticConstants {
public static final int TASK_QUEUE_CAP = 10;
+ public static final int AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = 70;
+
static {
SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER
+ ClusterNamespace.CLUSTER_DELIMITER +
FeConstants.INTERNAL_DB_NAME);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 2eac82c91b6..bcc7f53f4d1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -182,9 +182,13 @@ public class StatisticsAutoCollector extends
StatisticsCollector {
protected AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
TableIf table = StatisticsUtil
.findTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId);
+ // Skip tables that are too width.
+ if (table.getBaseSchema().size() >
StatisticsUtil.getAutoAnalyzeTableWidthThreshold()) {
+ return null;
+ }
+
AnalysisManager analysisManager =
Env.getServingEnv().getAnalysisManager();
TableStatsMeta tblStats =
analysisManager.findTableStatsStatus(table.getId());
-
if (!table.needReAnalyzeTable(tblStats)) {
return null;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 660cb874e9b..66b13ff87f1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -937,6 +937,16 @@ public class StatisticsUtil {
return StatisticConstants.ANALYZE_TIMEOUT_IN_SEC;
}
+ public static int getAutoAnalyzeTableWidthThreshold() {
+ try {
+ return
findConfigFromGlobalSessionVar(SessionVariable.AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD)
+ .autoAnalyzeTableWidthThreshold;
+ } catch (Exception e) {
+ LOG.warn("Failed to get value of
auto_analyze_table_width_threshold, return default", e);
+ }
+ return StatisticConstants.AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD;
+ }
+
public static String encodeValue(ResultRow row, int index) {
if (row == null || row.getValues().size() <= index) {
return "NULL";
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
index 14c6f41384f..732196ef31b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
@@ -221,6 +221,37 @@ public class StatisticsAutoCollectorTest {
Assertions.assertNotNull(statisticsAutoCollector.getReAnalyzeRequiredPart(analysisInfo2));
}
+ @Test
+ public void testSkipWideTable() {
+
+ TableIf tableIf = new OlapTable();
+
+ new MockUp<OlapTable>() {
+ @Mock
+ public List<Column> getBaseSchema() {
+ return Lists.newArrayList(new Column("col1", Type.INT), new
Column("col2", Type.INT));
+ }
+ };
+
+ new MockUp<StatisticsUtil>() {
+ int count = 0;
+ int [] thresholds = {1, 10};
+ @Mock
+ public TableIf findTable(long catalogName, long dbName, long
tblName) {
+ return tableIf;
+ }
+
+ @Mock
+ public int getAutoAnalyzeTableWidthThreshold() {
+ return thresholds[count++];
+ }
+ };
+ AnalysisInfo analysisInfo = new AnalysisInfoBuilder().build();
+ StatisticsAutoCollector statisticsAutoCollector = new
StatisticsAutoCollector();
+
Assertions.assertNull(statisticsAutoCollector.getReAnalyzeRequiredPart(analysisInfo));
+
Assertions.assertNotNull(statisticsAutoCollector.getReAnalyzeRequiredPart(analysisInfo));
+ }
+
@Test
public void testLoop() {
AtomicBoolean timeChecked = new AtomicBoolean();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
