This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new d38dcd77606 [fix](statistics)Refresh follower FE cache after alter
column stats. Support alter index column stats (#31108) (#31139)
d38dcd77606 is described below
commit d38dcd77606e267792d1d7151b1b2e4264e0e1ef
Author: Jibing-Li <[email protected]>
AuthorDate: Tue Feb 20 12:55:11 2024 +0800
[fix](statistics)Refresh follower FE cache after alter column stats.
Support alter index column stats (#31108) (#31139)
1. Refresh follower FE cache after alter column stats. So that follower
could update the cached stats too.
2. Support alter index column stats.
---
fe/fe-core/src/main/cup/sql_parser.cup | 7 +++-
.../doris/analysis/AlterColumnStatsStmt.java | 27 +++++++++++++-
.../org/apache/doris/statistics/ColStatsData.java | 12 +++++++
.../doris/statistics/StatisticsRepository.java | 11 +++---
.../suites/statistics/test_analyze_mv.groovy | 42 ++++++++++++++++++++++
5 files changed, 93 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index 7920c962efb..3f25b660ec9 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -1356,7 +1356,12 @@ alter_stmt ::=
| KW_ALTER KW_TABLE table_name:tbl KW_MODIFY KW_COLUMN ident:columnName
KW_SET KW_STATS LPAREN key_value_map:map RPAREN
opt_partition_names:partitionNames
{:
- RESULT = new AlterColumnStatsStmt(tbl, columnName, map,
partitionNames);
+ RESULT = new AlterColumnStatsStmt(tbl, null, columnName, map,
partitionNames);
+ :}
+ | KW_ALTER KW_TABLE table_name:tbl KW_INDEX ident:idx KW_MODIFY KW_COLUMN
ident:columnName
+ KW_SET KW_STATS LPAREN key_value_map:map RPAREN
opt_partition_names:partitionNames
+ {:
+ RESULT = new AlterColumnStatsStmt(tbl, idx, columnName, map,
partitionNames);
:}
| KW_ALTER KW_TABLE table_name:tbl KW_SET LPAREN key_value_map:properties
RPAREN
{:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColumnStatsStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColumnStatsStmt.java
index 7780f7833f7..ef4357dc285 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColumnStatsStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColumnStatsStmt.java
@@ -73,6 +73,7 @@ public class AlterColumnStatsStmt extends DdlStmt {
.build();
private final TableName tableName;
+ private final String indexName;
private final String columnName;
private final Map<String, String> properties;
private final PartitionNames optPartitionNames;
@@ -80,9 +81,12 @@ public class AlterColumnStatsStmt extends DdlStmt {
private final List<Long> partitionIds = Lists.newArrayList();
private final Map<StatsType, String> statsTypeToValue = Maps.newHashMap();
- public AlterColumnStatsStmt(TableName tableName, String columnName,
+ private long indexId = -1;
+
+ public AlterColumnStatsStmt(TableName tableName, String indexName, String
columnName,
Map<String, String> properties, PartitionNames optPartitionNames) {
this.tableName = tableName;
+ this.indexName = indexName;
this.columnName = columnName;
this.properties = properties == null ? Collections.emptyMap() :
properties;
this.optPartitionNames = optPartitionNames;
@@ -96,6 +100,10 @@ public class AlterColumnStatsStmt extends DdlStmt {
return columnName;
}
+ public long getIndexId() {
+ return indexId;
+ }
+
public List<Long> getPartitionIds() {
return partitionIds;
}
@@ -148,6 +156,19 @@ public class AlterColumnStatsStmt extends DdlStmt {
DatabaseIf db = catalog.getDbOrAnalysisException(tableName.getDb());
TableIf table = db.getTableOrAnalysisException(tableName.getTbl());
+ if (indexName != null) {
+ if (!(table instanceof OlapTable)) {
+ throw new AnalysisException("Only OlapTable support alter
index stats. "
+ + "Table " + table.getName() + " is not OlapTable.");
+ }
+ OlapTable olapTable = (OlapTable) table;
+ Long idxId = olapTable.getIndexIdByName(indexName);
+ if (idxId == null) {
+ throw new AnalysisException("Index " + indexName + " not exist
in table " + table.getName());
+ }
+ indexId = idxId;
+ }
+
if (table.getColumn(columnName) == null) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
columnName, FeNameFormat.getColumnNameRegex());
@@ -176,6 +197,10 @@ public class AlterColumnStatsStmt extends DdlStmt {
StringBuilder sb = new StringBuilder();
sb.append("ALTER TABLE ");
sb.append(tableName.toSql());
+ if (indexName != null) {
+ sb.append(" INDEX ");
+ sb.append(indexName);
+ }
sb.append(" MODIFY COLUMN ");
sb.append(columnName);
sb.append(" SET STATS ");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java
index bdc600987f4..c90b3dd8e1d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java
@@ -101,6 +101,18 @@ public class ColStatsData {
this.updateTime = row.get(13);
}
+ public ColStatsData(String id, long catalogId, long dbId, long tblId, long
idxId, String colId, String partId,
+ ColumnStatistic columnStatistic) {
+ this.statsId = new StatsId(id, catalogId, dbId, tblId, idxId, colId,
partId);
+ this.count = Math.round(columnStatistic.count);
+ this.ndv = Math.round(columnStatistic.ndv);
+ this.nullCount = Math.round(columnStatistic.numNulls);
+ this.minLit = columnStatistic.minExpr == null ? null :
columnStatistic.minExpr.getStringValue();
+ this.maxLit = columnStatistic.maxExpr == null ? null :
columnStatistic.maxExpr.getStringValue();
+ this.dataSizeInBytes = Math.round(columnStatistic.dataSize);
+ this.updateTime = columnStatistic.updatedTime;
+ }
+
public String toSQL(boolean roundByParentheses) {
StringJoiner sj = null;
if (roundByParentheses) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index d453a13df3c..3bf5b9fc019 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -249,6 +249,7 @@ public class StatisticsRepository {
String min = alterColumnStatsStmt.getValue(StatsType.MIN_VALUE);
String max = alterColumnStatsStmt.getValue(StatsType.MAX_VALUE);
String dataSize = alterColumnStatsStmt.getValue(StatsType.DATA_SIZE);
+ long indexId = alterColumnStatsStmt.getIndexId();
ColumnStatisticBuilder builder = new ColumnStatisticBuilder();
String colName = alterColumnStatsStmt.getColumnName();
Column column = objects.table.getColumn(colName);
@@ -284,10 +285,10 @@ public class StatisticsRepository {
ColumnStatistic columnStatistic = builder.build();
Map<String, String> params = new HashMap<>();
- params.put("id", constructId(objects.table.getId(), -1, colName));
+ params.put("id", constructId(objects.table.getId(), indexId, colName));
params.put("catalogId", String.valueOf(objects.catalog.getId()));
params.put("dbId", String.valueOf(objects.db.getId()));
- params.put("idxId", "-1");
+ params.put("idxId", String.valueOf(indexId));
params.put("tblId", String.valueOf(objects.table.getId()));
params.put("colId", String.valueOf(colName));
params.put("count", String.valueOf(columnStatistic.count));
@@ -301,8 +302,10 @@ public class StatisticsRepository {
// update table granularity statistics
params.put("partId", "NULL");
StatisticsUtil.execUpdate(INSERT_INTO_COLUMN_STATISTICS, params);
- Env.getCurrentEnv().getStatisticsCache()
- .updateColStatsCache(objects.table.getId(), -1, colName,
columnStatistic);
+ ColStatsData data = new
ColStatsData(constructId(objects.table.getId(), indexId, colName),
+ objects.catalog.getId(), objects.db.getId(),
objects.table.getId(), indexId, colName,
+ null, columnStatistic);
+ Env.getCurrentEnv().getStatisticsCache().syncColStats(data);
AnalysisInfo mockedJobInfo = new AnalysisInfoBuilder()
.setTblUpdateTime(System.currentTimeMillis())
.setColName("")
diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy
b/regression-test/suites/statistics/test_analyze_mv.groovy
index dfb12992e69..87af4bf9a43 100644
--- a/regression-test/suites/statistics/test_analyze_mv.groovy
+++ b/regression-test/suites/statistics/test_analyze_mv.groovy
@@ -409,6 +409,48 @@ suite("test_analyze_mv") {
verifyTaskStatus(result_sample, "mva_MIN__`value3`", "mv3")
verifyTaskStatus(result_sample, "mva_SUM__CAST(`value1` AS BIGINT)", "mv3")
+ // Test alter column stats
+ sql """drop stats mvTestDup"""
+ sql """alter table mvTestDup modify column key1 set stats ('ndv'='1',
'num_nulls'='1', 'min_value'='10', 'max_value'='40', 'row_count'='50');"""
+ sql """alter table mvTestDup index mv3 modify column mv_key1 set stats
('ndv'='5', 'num_nulls'='0', 'min_value'='0', 'max_value'='4',
'row_count'='5');"""
+ sql """alter table mvTestDup index mv3 modify column
`mva_SUM__CAST(``value1`` AS BIGINT)` set stats ('ndv'='10', 'num_nulls'='2',
'min_value'='1', 'max_value'='5', 'row_count'='11');"""
+
+ def result = sql """show column cached stats mvTestDup(key1)"""
+ assertEquals(1, result.size())
+ assertEquals("key1", result[0][0])
+ assertEquals("N/A", result[0][1])
+ assertEquals("50.0", result[0][2])
+ assertEquals("1.0", result[0][3])
+ assertEquals("1.0", result[0][4])
+ assertEquals("0.0", result[0][5])
+ assertEquals("0.0", result[0][6])
+ assertEquals("10", result[0][7])
+ assertEquals("40", result[0][8])
+
+ result = sql """show column cached stats mvTestDup(mv_key1)"""
+ assertEquals(1, result.size())
+ assertEquals("mv_key1", result[0][0])
+ assertEquals("mv3", result[0][1])
+ assertEquals("5.0", result[0][2])
+ assertEquals("5.0", result[0][3])
+ assertEquals("0.0", result[0][4])
+ assertEquals("0.0", result[0][5])
+ assertEquals("0.0", result[0][6])
+ assertEquals("0", result[0][7])
+ assertEquals("4", result[0][8])
+
+ result = sql """show column cached stats
mvTestDup(`mva_SUM__CAST(``value1`` AS BIGINT)`)"""
+ assertEquals(1, result.size())
+ assertEquals("mva_SUM__CAST(`value1` AS BIGINT)", result[0][0])
+ assertEquals("mv3", result[0][1])
+ assertEquals("11.0", result[0][2])
+ assertEquals("10.0", result[0][3])
+ assertEquals("2.0", result[0][4])
+ assertEquals("0.0", result[0][5])
+ assertEquals("0.0", result[0][6])
+ assertEquals("1", result[0][7])
+ assertEquals("5", result[0][8])
+
sql """drop database if exists test_analyze_mv"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
