This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ab3ac86e7c7 [fix](fe) Avoid eager allocations in cloud tablet indexes
(#66378)
ab3ac86e7c7 is described below
commit ab3ac86e7c7f30ac18625c13589b3df02aa8c1f4
Author: deardeng <[email protected]>
AuthorDate: Tue Aug 4 10:18:47 2026 +0800
[fix](fe) Avoid eager allocations in cloud tablet indexes (#66378)
Problem Summary: Cloud tablet route-cache rebuilding calls
fillBeToTablets for current and future placements. The previous
putIfAbsent calls eagerly constructed up to six candidate maps and sets
for every placement even when the keys already existed, and then
performed separate lookups. Use the containers returned by
computeIfAbsent at every level and add a focused unit test that verifies
all six container paths while preserving the global, table, and
partition index contents.
An isomorphic multi-scale JVM allocation model with 2 million entities,
four-route fan-out, current/future passes, and three index scopes
estimated cumulative allocation per modeled rebuild at 12.52 GiB before
and 8.59 GiB after the change, saving 3.93 GiB (31.4%). For 4 million
tablets, the fill-path cumulative-allocation reduction is expected to
remain about 31% under comparable topology, while the absolute GiB
saving depends on cluster and replica fan-out. The retained graph
remained about 3.76 GiB and the peak proxy about 966 MiB, so this change
does not claim a comparable steady-state heap or process-wide reduction.
Timing samples were noisy, so no throughput improvement is claimed.
---
.../doris/cloud/catalog/CloudTabletRebalancer.java | 21 +++----
.../cloud/catalog/CloudTabletRebalancerTest.java | 69 ++++++++++++++++++++++
2 files changed, 78 insertions(+), 12 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
index 6acaad24e07..1a231170b9e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
@@ -1059,22 +1059,19 @@ public class CloudTabletRebalancer extends MasterDaemon
{
ConcurrentHashMap<Long,
ConcurrentHashMap<Long, ConcurrentHashMap<Long, Set<Long>>>>
partToTablets) {
// global
- globalBeToTablets.putIfAbsent(be, ConcurrentHashMap.newKeySet());
- globalBeToTablets.get(be).add(tabletId);
+ globalBeToTablets.computeIfAbsent(be, ignored ->
ConcurrentHashMap.newKeySet()).add(tabletId);
// table
- beToTabletsInTable.putIfAbsent(tableId, new ConcurrentHashMap<Long,
Set<Long>>());
- ConcurrentHashMap<Long, Set<Long>> beToTabletsOfTable =
beToTabletsInTable.get(tableId);
- beToTabletsOfTable.putIfAbsent(be, ConcurrentHashMap.newKeySet());
- beToTabletsOfTable.get(be).add(tabletId);
+ ConcurrentHashMap<Long, Set<Long>> beToTabletsOfTable =
+ beToTabletsInTable.computeIfAbsent(tableId, ignored -> new
ConcurrentHashMap<>());
+ beToTabletsOfTable.computeIfAbsent(be, ignored ->
ConcurrentHashMap.newKeySet()).add(tabletId);
// partition
- partToTablets.putIfAbsent(partId, new ConcurrentHashMap<Long,
ConcurrentHashMap<Long, Set<Long>>>());
- ConcurrentHashMap<Long, ConcurrentHashMap<Long, Set<Long>>>
indexToTablets = partToTablets.get(partId);
- indexToTablets.putIfAbsent(indexId, new ConcurrentHashMap<Long,
Set<Long>>());
- ConcurrentHashMap<Long, Set<Long>> beToTabletsOfIndex =
indexToTablets.get(indexId);
- beToTabletsOfIndex.putIfAbsent(be, ConcurrentHashMap.newKeySet());
- beToTabletsOfIndex.get(be).add(tabletId);
+ ConcurrentHashMap<Long, ConcurrentHashMap<Long, Set<Long>>>
indexToTablets =
+ partToTablets.computeIfAbsent(partId, ignored -> new
ConcurrentHashMap<>());
+ ConcurrentHashMap<Long, Set<Long>> beToTabletsOfIndex =
+ indexToTablets.computeIfAbsent(indexId, ignored -> new
ConcurrentHashMap<>());
+ beToTabletsOfIndex.computeIfAbsent(be, ignored ->
ConcurrentHashMap.newKeySet()).add(tabletId);
}
private void enqueueWarmupTask(WarmupTabletTask task) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java
index 8637be3c76a..080ee4f5a34 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java
@@ -41,6 +41,7 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
public class CloudTabletRebalancerTest {
@@ -81,6 +82,30 @@ public class CloudTabletRebalancerTest {
}
}
+ private static class CountingConcurrentHashMap<K, V> extends
ConcurrentHashMap<K, V> {
+ private int computeIfAbsentCalls;
+ private int getCalls;
+ private int putIfAbsentCalls;
+
+ @Override
+ public V computeIfAbsent(K key, Function<? super K, ? extends V>
mappingFunction) {
+ computeIfAbsentCalls++;
+ return super.computeIfAbsent(key, mappingFunction);
+ }
+
+ @Override
+ public V get(Object key) {
+ getCalls++;
+ return super.get(key);
+ }
+
+ @Override
+ public V putIfAbsent(K key, V value) {
+ putIfAbsentCalls++;
+ return super.putIfAbsent(key, value);
+ }
+ }
+
private static void setField(Object obj, String name, Object value) throws
Exception {
Field f = CloudTabletRebalancer.class.getDeclaredField(name);
f.setAccessible(true);
@@ -101,6 +126,50 @@ public class CloudTabletRebalancerTest {
return (T) m.invoke(obj, args);
}
+ @Test
+ public void testFillBeToTabletsUsesComputedContainers() {
+ TestRebalancer rebalancer = new TestRebalancer();
+ long beId = 1L;
+ long tableId = 2L;
+ long partitionId = 3L;
+ long indexId = 4L;
+
+ CountingConcurrentHashMap<Long, Set<Long>> globalBeToTablets = new
CountingConcurrentHashMap<>();
+ CountingConcurrentHashMap<Long, ConcurrentHashMap<Long, Set<Long>>>
beToTabletsInTable =
+ new CountingConcurrentHashMap<>();
+ CountingConcurrentHashMap<Long, Set<Long>> beToTabletsOfTable = new
CountingConcurrentHashMap<>();
+ beToTabletsInTable.put(tableId, beToTabletsOfTable);
+
+ CountingConcurrentHashMap<Long, ConcurrentHashMap<Long,
ConcurrentHashMap<Long, Set<Long>>>>
+ partToTablets = new CountingConcurrentHashMap<>();
+ CountingConcurrentHashMap<Long, ConcurrentHashMap<Long, Set<Long>>>
indexToTablets =
+ new CountingConcurrentHashMap<>();
+ CountingConcurrentHashMap<Long, Set<Long>> beToTabletsOfIndex = new
CountingConcurrentHashMap<>();
+ partToTablets.put(partitionId, indexToTablets);
+ indexToTablets.put(indexId, beToTabletsOfIndex);
+
+ rebalancer.fillBeToTablets(beId, tableId, partitionId, indexId, 5L,
+ globalBeToTablets, beToTabletsInTable, partToTablets);
+ rebalancer.fillBeToTablets(beId, tableId, partitionId, indexId, 6L,
+ globalBeToTablets, beToTabletsInTable, partToTablets);
+
+ assertComputedContainerUsed(globalBeToTablets);
+ assertComputedContainerUsed(beToTabletsInTable);
+ assertComputedContainerUsed(beToTabletsOfTable);
+ assertComputedContainerUsed(partToTablets);
+ assertComputedContainerUsed(indexToTablets);
+ assertComputedContainerUsed(beToTabletsOfIndex);
+ Assertions.assertEquals(Set.of(5L, 6L), globalBeToTablets.get(beId));
+ Assertions.assertEquals(Set.of(5L, 6L), beToTabletsOfTable.get(beId));
+ Assertions.assertEquals(Set.of(5L, 6L), beToTabletsOfIndex.get(beId));
+ }
+
+ private static void
assertComputedContainerUsed(CountingConcurrentHashMap<?, ?> map) {
+ Assertions.assertEquals(2, map.computeIfAbsentCalls);
+ Assertions.assertEquals(0, map.putIfAbsentCalls);
+ Assertions.assertEquals(0, map.getCalls);
+ }
+
@Test
public void testPickTabletPreferCold_picksColdWhenAvailable() throws
Exception {
TestRebalancer r = new TestRebalancer();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
