This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 596ef34f1f5 [fix](statistics)Fix stats cleaner delete partition stats
bug (#30648)
596ef34f1f5 is described below
commit 596ef34f1f59944a9be8dc08e6d8050d431779da
Author: Jibing-Li <[email protected]>
AuthorDate: Wed Jan 31 20:50:33 2024 +0800
[fix](statistics)Fix stats cleaner delete partition stats bug (#30648)
When a partition in OlapTable is removed, we should use partition id to
delete the related stats record in column_statistics. Before, it was using id,
which may cause delete useful stats of other partition.
---
.../java/org/apache/doris/statistics/StatisticsCleaner.java | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
index 1af219baa6f..3a142351a28 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
@@ -169,12 +169,15 @@ public class StatisticsCleaner extends MasterDaemon {
doDelete("idx_id", expiredStats.expiredIdxId.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
+ doDelete("part_id", expiredStats.expiredPartitionId.stream()
+ .map(String::valueOf).collect(Collectors.toList()),
+ FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
doDelete("id", expiredStats.ids.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
}
- private void doDelete(String/*col name*/ colName, List<String> pred,
String tblName, boolean taskOnly) {
+ private void doDelete(String colName, List<String> pred, String tblName,
boolean taskOnly) {
String deleteTemplate = "DELETE FROM " + tblName + " WHERE ${left} IN
(${right})";
if (CollectionUtils.isEmpty(pred)) {
return;
@@ -241,7 +244,7 @@ public class StatisticsCleaner extends MasterDaemon {
continue;
}
if
(!olapTable.getPartitionIds().contains(Long.parseLong(partId))) {
- expiredStats.ids.add(id);
+
expiredStats.expiredPartitionId.add(Long.parseLong(partId));
}
} catch (Exception e) {
LOG.warn("Error occurred when retrieving expired stats",
e);
@@ -256,9 +259,8 @@ public class StatisticsCleaner extends MasterDaemon {
Set<Long> expiredCatalog = new HashSet<>();
Set<Long> expiredDatabase = new HashSet<>();
Set<Long> expiredTable = new HashSet<>();
-
Set<Long> expiredIdxId = new HashSet<>();
-
+ Set<Long> expiredPartitionId = new HashSet<>();
Set<String> ids = new HashSet<>();
public boolean isFull() {
@@ -266,6 +268,7 @@ public class StatisticsCleaner extends MasterDaemon {
|| expiredDatabase.size() >=
Config.max_allowed_in_element_num_of_delete
|| expiredTable.size() >=
Config.max_allowed_in_element_num_of_delete
|| expiredIdxId.size() >=
Config.max_allowed_in_element_num_of_delete
+ || expiredPartitionId.size() >=
Config.max_allowed_in_element_num_of_delete
|| ids.size() >=
Config.max_allowed_in_element_num_of_delete;
}
@@ -274,6 +277,7 @@ public class StatisticsCleaner extends MasterDaemon {
&& expiredDatabase.isEmpty()
&& expiredTable.isEmpty()
&& expiredIdxId.isEmpty()
+ && expiredPartitionId.isEmpty()
&& ids.size() <
Config.max_allowed_in_element_num_of_delete / 10;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
