This is an automated email from the ASF dual-hosted git repository.
xy720 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8d0eacd574c [fix](catalog) Fix FE memory leak of storage policy
entries persisted in image by dropped partitions (#68541)
8d0eacd574c is described below
commit 8d0eacd574c37c45f8bc8ad99f053e9ec8b564bb
Author: xy720 <[email protected]>
AuthorDate: Sun Oct 11 16:35:03 2026 +0800
[fix](catalog) Fix FE memory leak of storage policy entries persisted in
image by dropped partitions (#68541)
Fix an FE memory leak where storage policy entries of dropped
partitions, persisted in the image by older versions, were never
released and could cause FE OOM.
---
.../org/apache/doris/catalog/PartitionInfo.java | 20 +++-
.../apache/doris/catalog/PartitionInfoTest.java | 108 +++++++++++++++++++++
2 files changed, 127 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index 3aaab1b597a..f79af5acf41 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -28,6 +28,7 @@ import org.apache.doris.analysis.TimeStampNsLiteral;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
+import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TTabletType;
@@ -39,6 +40,7 @@ import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -51,7 +53,7 @@ import java.util.stream.Collectors;
/*
* Repository of a partition's related infos. should modify only under table's
lock.
*/
-public class PartitionInfo {
+public class PartitionInfo implements GsonPostProcessable {
private static final Logger LOG =
LogManager.getLogger(PartitionInfo.class);
@SerializedName("Type")
@@ -405,6 +407,22 @@ public class PartitionInfo {
idToInMemory.put(partitionId, isInMemory);
}
+ @Override
+ public void gsonPostProcess() throws IOException {
+ // Every live partition has a data property, so storage policy entries
without one
+ // belong to dropped partitions.
+ if (idToStoragePolicy == null || idToDataProperty == null) {
+ return;
+ }
+ int before = idToStoragePolicy.size();
+ idToStoragePolicy.keySet().retainAll(idToDataProperty.keySet());
+ int removed = before - idToStoragePolicy.size();
+ if (removed > 0) {
+ LOG.info("removed {} stale storage policy entries of dropped
partitions, remaining {}",
+ removed, idToStoragePolicy.size());
+ }
+ }
+
public boolean isMultiColumnPartition() {
return isMultiColumnPartition;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionInfoTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionInfoTest.java
new file mode 100644
index 00000000000..4bc42f300fb
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionInfoTest.java
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.analysis.PartitionKeyDesc;
+import org.apache.doris.analysis.PartitionValue;
+import org.apache.doris.analysis.SinglePartitionDesc;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.thrift.TTabletType;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class PartitionInfoTest {
+ private ListPartitionInfo partitionInfo;
+
+ @BeforeEach
+ public void setUp() {
+ Column k1 = new Column("k1", new ScalarType(PrimitiveType.INT), true,
null, "", "");
+ partitionInfo = new ListPartitionInfo(Lists.newArrayList(k1));
+ }
+
+ private void addPartition(long partitionId, String value, boolean isTemp)
+ throws AnalysisException, DdlException {
+ List<List<PartitionValue>> inValues = new ArrayList<>();
+ inValues.add(Lists.newArrayList(new PartitionValue(value)));
+ SinglePartitionDesc desc = new SinglePartitionDesc(false, "p" +
partitionId,
+ PartitionKeyDesc.createIn(inValues), null);
+ desc.analyze(1, null);
+ partitionInfo.handleNewSinglePartitionDesc(desc, partitionId, isTemp);
+ }
+
+ @Test
+ public void testDropPartitionRemovesAllPartitionScopedEntries() throws
Exception {
+ addPartition(1L, "1", false);
+ partitionInfo.setTabletType(1L, TTabletType.TABLET_TYPE_DISK);
+ Assertions.assertTrue(partitionInfo.idToStoragePolicy.containsKey(1L));
+ Assertions.assertTrue(partitionInfo.idToTabletType.containsKey(1L));
+
+ partitionInfo.dropPartition(1L);
+
+ Assertions.assertTrue(partitionInfo.idToDataProperty.isEmpty());
+ Assertions.assertTrue(partitionInfo.idToStoragePolicy.isEmpty());
+
Assertions.assertTrue(partitionInfo.idToInvertedIndexFileStorageFormat.isEmpty());
+ Assertions.assertTrue(partitionInfo.idToReplicaAllocation.isEmpty());
+ Assertions.assertTrue(partitionInfo.idToInMemory.isEmpty());
+ Assertions.assertTrue(partitionInfo.idToTabletType.isEmpty());
+ Assertions.assertTrue(partitionInfo.idToItem.isEmpty());
+ }
+
+ @Test
+ public void testRepeatedOverwriteDoesNotAccumulateStoragePolicy() throws
Exception {
+ long formalId = 1L;
+ addPartition(formalId, "1", false);
+ long nextId = 100L;
+ for (int i = 0; i < 1000; i++) {
+ long tempId = nextId++;
+ addPartition(tempId, "1", true);
+ partitionInfo.dropPartition(formalId);
+ partitionInfo.moveFromTempToFormal(tempId);
+ formalId = tempId;
+ }
+
+ Assertions.assertEquals(Sets.newHashSet(formalId),
partitionInfo.idToStoragePolicy.keySet());
+ Assertions.assertEquals(Sets.newHashSet(formalId),
partitionInfo.idToDataProperty.keySet());
+ }
+
+ @Test
+ public void testGsonPostProcessRemovesStaleStoragePolicy() throws
Exception {
+ addPartition(1L, "1", false);
+ addPartition(2L, "2", false);
+ partitionInfo.setStoragePolicy(2L, "policy_a");
+ partitionInfo.idToStoragePolicy.put(1000L, "");
+ partitionInfo.idToStoragePolicy.put(1001L, "policy_b");
+
+ String json = GsonUtils.GSON.toJson(partitionInfo,
PartitionInfo.class);
+ PartitionInfo restored = GsonUtils.GSON.fromJson(json,
PartitionInfo.class);
+
+ Assertions.assertTrue(restored instanceof ListPartitionInfo);
+ Assertions.assertEquals(Sets.newHashSet(1L, 2L),
restored.idToStoragePolicy.keySet());
+ Assertions.assertEquals("", restored.getStoragePolicy(1L));
+ Assertions.assertEquals("policy_a", restored.getStoragePolicy(2L));
+ Assertions.assertEquals(Sets.newHashSet(1L, 2L),
restored.idToDataProperty.keySet());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
