This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new c0b3425a644 [fix](statistics)Fix stats cleaner delete partition stats
bug (#30648) (#30665)
c0b3425a644 is described below
commit c0b3425a64461bcbb217da1fa53063a23f090590
Author: Jibing-Li <[email protected]>
AuthorDate: Thu Feb 1 10:07:09 2024 +0800
[fix](statistics)Fix stats cleaner delete partition stats bug (#30648)
(#30665)
When a partition in OlapTable is removed, we should use partition id to
delete the related stats record in column_statistics. Before, it was using id,
which may cause delete useful stats of other partition.
---
.../java/org/apache/doris/statistics/StatisticsCleaner.java | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
index 21deb44c0a4..388f7742a95 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
@@ -170,12 +170,15 @@ public class StatisticsCleaner extends MasterDaemon {
doDelete("idx_id", expiredStats.expiredIdxId.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
+ doDelete("part_id", expiredStats.expiredPartitionId.stream()
+ .map(String::valueOf).collect(Collectors.toList()),
+ FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
doDelete("id", expiredStats.ids.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
}
- private void doDelete(String/*col name*/ colName, List<String> pred,
String tblName, boolean taskOnly) {
+ private void doDelete(String colName, List<String> pred, String tblName,
boolean taskOnly) {
String deleteTemplate = "DELETE FROM " + tblName + " WHERE ${left} IN
(${right})";
if (CollectionUtils.isEmpty(pred)) {
return;
@@ -242,7 +245,7 @@ public class StatisticsCleaner extends MasterDaemon {
continue;
}
if
(!olapTable.getPartitionIds().contains(Long.parseLong(partId))) {
- expiredStats.ids.add(id);
+
expiredStats.expiredPartitionId.add(Long.parseLong(partId));
}
} catch (Exception e) {
LOG.warn("Error occurred when retrieving expired stats",
e);
@@ -257,9 +260,8 @@ public class StatisticsCleaner extends MasterDaemon {
Set<Long> expiredCatalog = new HashSet<>();
Set<Long> expiredDatabase = new HashSet<>();
Set<Long> expiredTable = new HashSet<>();
-
Set<Long> expiredIdxId = new HashSet<>();
-
+ Set<Long> expiredPartitionId = new HashSet<>();
Set<String> ids = new HashSet<>();
public boolean isFull() {
@@ -267,6 +269,7 @@ public class StatisticsCleaner extends MasterDaemon {
|| expiredDatabase.size() >=
Config.max_allowed_in_element_num_of_delete
|| expiredTable.size() >=
Config.max_allowed_in_element_num_of_delete
|| expiredIdxId.size() >=
Config.max_allowed_in_element_num_of_delete
+ || expiredPartitionId.size() >=
Config.max_allowed_in_element_num_of_delete
|| ids.size() >=
Config.max_allowed_in_element_num_of_delete;
}
@@ -275,6 +278,7 @@ public class StatisticsCleaner extends MasterDaemon {
&& expiredDatabase.isEmpty()
&& expiredTable.isEmpty()
&& expiredIdxId.isEmpty()
+ && expiredPartitionId.isEmpty()
&& ids.size() <
Config.max_allowed_in_element_num_of_delete / 10;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
