This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4ff165e6bf1 [fix](statistics)Partition stats support truncate single
partition. (#36758)
4ff165e6bf1 is described below
commit 4ff165e6bf1ff5ee41b8ffe0e39f853e56ad278c
Author: Jibing-Li <[email protected]>
AuthorDate: Tue Jun 25 10:25:34 2024 +0800
[fix](statistics)Partition stats support truncate single partition. (#36758)
Partition stats support truncate single partition. When truncate single
partition, drop the stats of this partition and trigger auto analyze in
next round.
---
.../apache/doris/datasource/InternalCatalog.java | 8 +--
.../apache/doris/statistics/AnalysisManager.java | 13 ++--
.../suites/statistics/test_partition_stats.groovy | 79 ++++++++++++++++++++++
3 files changed, 92 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 685e0f5e8c2..d349b725770 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -44,6 +44,7 @@ import org.apache.doris.analysis.KeysDesc;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.PartitionDesc;
import org.apache.doris.analysis.PartitionKeyDesc;
+import org.apache.doris.analysis.PartitionNames;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.RecoverDbStmt;
import org.apache.doris.analysis.RecoverPartitionStmt;
@@ -3439,10 +3440,9 @@ public class InternalCatalog implements
CatalogIf<Database> {
erasePartitionDropBackendReplicas(oldPartitions);
- if (truncateEntireTable) {
- // Drop the whole table stats after truncate the entire table
- Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable);
- }
+ PartitionNames partitionNames = truncateEntireTable ? null
+ : new PartitionNames(false,
tblRef.getPartitionNames().getPartitionNames());
+ Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable,
partitionNames);
Env.getCurrentEnv().getAnalysisManager().updateUpdatedRows(updateRecords,
db.getId(), olapTable.getId());
LOG.info("finished to truncate table {}, partitions: {}",
tblRef.getName().toSql(), tblRef.getPartitionNames());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index d2deade518d..22bc11971c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -668,7 +668,7 @@ public class AnalysisManager implements Writable {
StatisticsRepository.dropStatistics(catalogId, dbId, tblId, cols,
partitions);
}
- public void dropStats(TableIf table) throws DdlException {
+ public void dropStats(TableIf table, PartitionNames partitionNames) throws
DdlException {
TableStatsMeta tableStats = findTableStatsStatus(table.getId());
if (tableStats == null) {
return;
@@ -676,10 +676,15 @@ public class AnalysisManager implements Writable {
long catalogId = table.getDatabase().getCatalog().getId();
long dbId = table.getDatabase().getId();
long tableId = table.getId();
- invalidateLocalStats(catalogId, dbId, tableId, null, tableStats, null);
+ invalidateLocalStats(catalogId, dbId, tableId, null, tableStats,
partitionNames);
// Drop stats ddl is master only operation.
- invalidateRemoteStats(catalogId, dbId, tableId, null, null);
- StatisticsRepository.dropStatistics(catalogId, dbId, table.getId(),
null, null);
+ Set<String> partitions = null;
+ if (partitionNames != null && !partitionNames.isStar() &&
partitionNames.getPartitionNames() != null) {
+ partitions = new HashSet<>(partitionNames.getPartitionNames());
+ }
+ // Drop stats ddl is master only operation.
+ invalidateRemoteStats(catalogId, dbId, tableId, null, partitions);
+ StatisticsRepository.dropStatistics(catalogId, dbId, table.getId(),
null, partitions);
}
public void invalidateLocalStats(long catalogId, long dbId, long tableId,
Set<String> columns,
diff --git a/regression-test/suites/statistics/test_partition_stats.groovy
b/regression-test/suites/statistics/test_partition_stats.groovy
index 1f4962d941d..02815c3235c 100644
--- a/regression-test/suites/statistics/test_partition_stats.groovy
+++ b/regression-test/suites/statistics/test_partition_stats.groovy
@@ -736,6 +736,85 @@ suite("test_partition_stats") {
result = sql """show column stats part7 partition(*)"""
assertEquals(27, result.size())
+ // Test truncate table
+ sql """truncate table part7 partition(p1)"""
+ result = sql """show table stats part7 partition(*)"""
+ assertEquals(2, result.size())
+ assertNotEquals('p1', result[0][0])
+ assertNotEquals('p1', result[1][0])
+ result = sql """show column stats part7 partition(p1)"""
+ assertEquals(0, result.size())
+ result = sql """show column cached stats part7 partition(p1)"""
+ assertEquals(0, result.size())
+ result = sql """show column stats part7 partition(p2)"""
+ assertEquals(9, result.size())
+ result = sql """show column stats part7 partition(p3)"""
+ assertEquals(9, result.size())
+
+ sql """Insert into part7 values (7, 7, 7, 7, 7, 7, 7.7, 7.7, 7.7)"""
+ result = sql """show table stats part7 partition(p1)"""
+ assertEquals(1, result.size())
+ assertEquals("p1", result[0][0])
+ assertEquals("1", result[0][1])
+ sql """analyze table part7 properties("use.auto.analyzer"="true")"""
+ result = sql """show column stats part7"""
+ assertEquals(9, result.size())
+ assertEquals("17.0", result[0][2])
+ assertEquals("17.0", result[1][2])
+ assertEquals("17.0", result[2][2])
+ assertEquals("17.0", result[3][2])
+ assertEquals("17.0", result[4][2])
+ assertEquals("17.0", result[5][2])
+ assertEquals("17.0", result[6][2])
+ assertEquals("17.0", result[7][2])
+ assertEquals("17.0", result[8][2])
+ result = sql """show column cached stats part7"""
+ assertEquals(9, result.size())
+ assertEquals("17.0", result[0][2])
+ assertEquals("17.0", result[1][2])
+ assertEquals("17.0", result[2][2])
+ assertEquals("17.0", result[3][2])
+ assertEquals("17.0", result[4][2])
+ assertEquals("17.0", result[5][2])
+ assertEquals("17.0", result[6][2])
+ assertEquals("17.0", result[7][2])
+ assertEquals("17.0", result[8][2])
+ result = sql """show column stats part7 partition(p1)"""
+ assertEquals(9, result.size())
+ result = sql """show column cached stats part7 partition(p1)"""
+ assertEquals(9, result.size())
+
+ sql """truncate table part7"""
+ result = sql """show table stats part7 partition(*)"""
+ assertEquals(0, result.size())
+ result = sql """show column stats part7"""
+ assertEquals(0, result.size())
+ result = sql """show column cached stats part7"""
+ assertEquals(0, result.size())
+ sql """analyze table part7 properties("use.auto.analyzer"="true")"""
+ result = sql """show column stats part7"""
+ assertEquals(9, result.size())
+ assertEquals("0.0", result[0][2])
+ assertEquals("0.0", result[1][2])
+ assertEquals("0.0", result[2][2])
+ assertEquals("0.0", result[3][2])
+ assertEquals("0.0", result[4][2])
+ assertEquals("0.0", result[5][2])
+ assertEquals("0.0", result[6][2])
+ assertEquals("0.0", result[7][2])
+ assertEquals("0.0", result[8][2])
+ result = sql """show column cached stats part7"""
+ assertEquals(9, result.size())
+ assertEquals("0.0", result[0][2])
+ assertEquals("0.0", result[1][2])
+ assertEquals("0.0", result[2][2])
+ assertEquals("0.0", result[3][2])
+ assertEquals("0.0", result[4][2])
+ assertEquals("0.0", result[5][2])
+ assertEquals("0.0", result[6][2])
+ assertEquals("0.0", result[7][2])
+ assertEquals("0.0", result[8][2])
+
sql """drop database test_partition_stats"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
