This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0b495134fcf [fix](fe) modify TabletInvertedIndex to reduce memory
(#61080)
0b495134fcf is described below
commit 0b495134fcf3271f5a8b1e4f02342bff6b3b7791
Author: meiyi <[email protected]>
AuthorDate: Thu Mar 12 15:36:09 2026 +0800
[fix](fe) modify TabletInvertedIndex to reduce memory (#61080)
### What problem does this PR solve?
for about 1340000 tablets in cloud mode, the fe memory is:
before:
<img width="1834" height="174" alt="image"
src="https://github.com/user-attachments/assets/f2392442-bb4c-4a85-b506-559400932b8e";
/>
after:
<img width="1842" height="312" alt="image"
src="https://github.com/user-attachments/assets/f3e1351f-3bc8-4994-89fb-8e423f77aff9";
/>
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
dist/LICENSE-dist.txt | 2 +-
fe/fe-core/pom.xml | 5 ++
.../doris/catalog/LocalTabletInvertedIndex.java | 61 +++++++++++++++-------
.../apache/doris/catalog/TabletInvertedIndex.java | 4 +-
.../cloud/catalog/CloudTabletInvertedIndex.java | 5 +-
fe/pom.xml | 8 ++-
6 files changed, 59 insertions(+), 26 deletions(-)
diff --git a/dist/LICENSE-dist.txt b/dist/LICENSE-dist.txt
index 0aabefe4df5..b0432407456 100644
--- a/dist/LICENSE-dist.txt
+++ b/dist/LICENSE-dist.txt
@@ -1085,7 +1085,7 @@ The Apache Software License, Version 2.0
* fastjson:
- com.alibaba:fastjson:1.2.58 (https://github.com/alibaba/fastjson)
* fastutil:
- - it.unimi.dsi:fastutil:6.5.6 (http://fasutil.dsi.unimi.it/)
+ - it.unimi.dsi:fastutil:8.5.18 (http://fastutil.di.unimi.it/)
* fe-common:
- org.apache.doris:fe-common:0.15-SNAPSHOT
(https://doris.apache.org/fe-common/)
* hadoop-mapreduce-client-app:
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index f1f4932381b..8387cffbe3d 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -801,6 +801,11 @@ under the License.
<artifactId>mockito-inline</artifactId>
<scope>test</scope>
</dependency>
+ <!-- fastutil for memory-efficient primitive collections -->
+ <dependency>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil-core</artifactId>
+ </dependency>
</dependencies>
<repositories>
<!-- for hive-catalog-shade -->
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java
index 6d47592c9e9..27d1b6ba675 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java
@@ -48,10 +48,12 @@ import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
import com.google.common.collect.Table;
import com.google.common.collect.TreeMultimap;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -71,7 +73,8 @@ public class LocalTabletInvertedIndex extends
TabletInvertedIndex {
// backing replica table, for visiting backend replicas faster.
// backend id -> (tablet id -> replica)
- private Table<Long, Long, Replica> backingReplicaMetaTable =
HashBasedTable.create();
+ private Long2ObjectOpenHashMap<Long2ObjectOpenHashMap<Replica>>
backingReplicaMetaTable
+ = new Long2ObjectOpenHashMap<>();
// partition id -> partition info.
// notice partition info update every
Config.partition_info_update_interval_secs seconds,
@@ -116,13 +119,15 @@ public class LocalTabletInvertedIndex extends
TabletInvertedIndex {
LOG.debug("begin to do tablet diff with backend[{}]. num: {}",
backendId, backendTablets.size());
}
- Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.row(backendId);
+ Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.get(backendId);
if (replicaMetaWithBackend != null) {
feTabletNum = replicaMetaWithBackend.size();
processTabletReportAsync(backendId, backendTablets,
backendPartitionsVersion, storageMediumMap,
tabletSyncMap, tabletDeleteFromMeta,
tabletFoundInMeta, tabletMigrationMap,
partitionVersionSyncMap, transactionsToPublish,
transactionsToClear, tabletRecoveryMap,
tabletToUpdate, cooldownTablets,
replicaMetaWithBackend);
+ } else {
+ processPartitionVersions(backendPartitionsVersion,
partitionVersionSyncMap);
}
} finally {
readUnlock(stamp);
@@ -726,7 +731,13 @@ public class LocalTabletInvertedIndex extends
TabletInvertedIndex {
Map<Long, Replica> replicas =
replicaMetaTable.rowMap().remove(tabletId);
if (replicas != null) {
for (long backendId : replicas.keySet()) {
- backingReplicaMetaTable.remove(backendId, tabletId);
+ Map<Long, Replica> backendMap =
backingReplicaMetaTable.get(backendId);
+ if (backendMap != null) {
+ backendMap.remove(tabletId);
+ if (backendMap.isEmpty()) {
+ backingReplicaMetaTable.remove(backendId);
+ }
+ }
}
}
tabletMetaMap.remove(tabletId);
@@ -747,7 +758,8 @@ public class LocalTabletInvertedIndex extends
TabletInvertedIndex {
"tablet " + tabletId + " not exists, replica " +
replica.getId()
+ ", backend " + backendId);
replicaMetaTable.put(tabletId, backendId, replica);
- backingReplicaMetaTable.put(backendId, tabletId, replica);
+ backingReplicaMetaTable.computeIfAbsent(backendId, k -> new
Long2ObjectOpenHashMap<>())
+ .put(tabletId, replica);
if (LOG.isDebugEnabled()) {
LOG.debug("add replica {} of tablet {} in backend {}",
replica.getId(), tabletId, backendId);
@@ -766,10 +778,17 @@ public class LocalTabletInvertedIndex extends
TabletInvertedIndex {
if (replicaMetaTable.containsRow(tabletId)) {
Replica replica = replicaMetaTable.remove(tabletId, backendId);
- backingReplicaMetaTable.remove(backendId, tabletId);
if (LOG.isDebugEnabled()) {
LOG.debug("delete replica {} of tablet {} in backend {}",
- replica.getId(), tabletId, backendId);
+ replica == null ? null : replica.getId(),
tabletId, backendId);
+ }
+
+ Map<Long, Replica> backendMap =
backingReplicaMetaTable.get(backendId);
+ if (backendMap != null) {
+ backendMap.remove(tabletId);
+ if (backendMap.isEmpty()) {
+ backingReplicaMetaTable.remove(backendId);
+ }
}
} else {
// this may happen when fe restart after tablet is empty(bug
cause)
@@ -800,7 +819,7 @@ public class LocalTabletInvertedIndex extends
TabletInvertedIndex {
if (replicaMetaTable.containsRow(tabletId)) {
return
Lists.newArrayList(replicaMetaTable.row(tabletId).values());
}
- return Lists.newArrayList();
+ return Collections.emptyList();
} finally {
readUnlock(stamp);
}
@@ -811,7 +830,7 @@ public class LocalTabletInvertedIndex extends
TabletInvertedIndex {
Long ret = 0L;
long stamp = readLock();
try {
- Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.row(backendId);
+ Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.get(backendId);
if (replicaMetaWithBackend != null) {
ret += replicaMetaWithBackend.size();
}
@@ -823,28 +842,26 @@ public class LocalTabletInvertedIndex extends
TabletInvertedIndex {
@Override
public List<Long> getTabletIdsByBackendId(long backendId) {
- List<Long> tabletIds = Lists.newArrayList();
long stamp = readLock();
try {
- Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.row(backendId);
+ Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.get(backendId);
if (replicaMetaWithBackend != null) {
- tabletIds.addAll(replicaMetaWithBackend.keySet());
+ return Lists.newArrayList(replicaMetaWithBackend.keySet());
}
} finally {
readUnlock(stamp);
}
- return tabletIds;
+ return Collections.emptyList();
}
@Override
public List<Pair<Long, Long>>
getTabletSizeByBackendIdAndStorageMedium(long backendId,
TStorageMedium storageMedium) {
- List<Pair<Long, Long>> tabletIdSizes = Lists.newArrayList();
long stamp = readLock();
try {
- Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.row(backendId);
+ Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.get(backendId);
if (replicaMetaWithBackend != null) {
- tabletIdSizes = replicaMetaWithBackend.entrySet().stream()
+ return replicaMetaWithBackend.entrySet().stream()
.filter(entry ->
tabletMetaMap.get(entry.getKey()).getStorageMedium() == storageMedium)
.map(entry -> Pair.of(entry.getKey(),
entry.getValue().getDataSize()))
.collect(Collectors.toList());
@@ -852,14 +869,14 @@ public class LocalTabletInvertedIndex extends
TabletInvertedIndex {
} finally {
readUnlock(stamp);
}
- return tabletIdSizes;
+ return Collections.emptyList();
}
@Override
public int getTabletNumByBackendId(long backendId) {
long stamp = readLock();
try {
- Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.row(backendId);
+ Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.get(backendId);
if (replicaMetaWithBackend != null) {
return replicaMetaWithBackend.size();
}
@@ -876,7 +893,7 @@ public class LocalTabletInvertedIndex extends
TabletInvertedIndex {
long ssdNum = 0;
long stamp = readLock();
try {
- Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.row(backendId);
+ Map<Long, Replica> replicaMetaWithBackend =
backingReplicaMetaTable.get(backendId);
if (replicaMetaWithBackend != null) {
for (long tabletId : replicaMetaWithBackend.keySet()) {
if (tabletMetaMap.get(tabletId).getStorageMedium() ==
TStorageMedium.HDD) {
@@ -1033,7 +1050,13 @@ public class LocalTabletInvertedIndex extends
TabletInvertedIndex {
public Table<Long, Long, Replica> getBackingReplicaMetaTable() {
long stamp = readLock();
try {
- return HashBasedTable.create(backingReplicaMetaTable);
+ Table<Long, Long, Replica> table = HashBasedTable.create();
+ backingReplicaMetaTable.entrySet().forEach(backendEntry -> {
+ backendEntry.getValue().entrySet().forEach(replicaEntry -> {
+ table.put(backendEntry.getKey(), replicaEntry.getKey(),
replicaEntry.getValue());
+ });
+ });
+ return table;
} finally {
readUnlock(stamp);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
index 71b99a0a862..420c1087066 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
@@ -28,10 +28,10 @@ import org.apache.doris.thrift.TTabletMetaInfo;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ListMultimap;
-import com.google.common.collect.Maps;
import com.google.common.collect.SetMultimap;
import com.google.common.collect.Table;
import com.google.common.collect.TreeMultimap;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -60,7 +60,7 @@ public abstract class TabletInvertedIndex {
private StampedLock lock = new StampedLock();
// tablet id -> tablet meta
- protected Map<Long, TabletMeta> tabletMetaMap = Maps.newHashMap();
+ protected Long2ObjectOpenHashMap<TabletMeta> tabletMetaMap = new
Long2ObjectOpenHashMap<>();
public TabletInvertedIndex() {
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletInvertedIndex.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletInvertedIndex.java
index af6a368f3f9..6c68429f9c0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletInvertedIndex.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletInvertedIndex.java
@@ -21,20 +21,19 @@ import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.TabletInvertedIndex;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
public class CloudTabletInvertedIndex extends TabletInvertedIndex {
private static final Logger LOG =
LogManager.getLogger(CloudTabletInvertedIndex.class);
// tablet id -> replica
// for cloud mode, no need to know the replica's backend
- private Map<Long, Replica> replicaMetaMap = Maps.newHashMap();
+ private Long2ObjectOpenHashMap<Replica> replicaMetaMap = new
Long2ObjectOpenHashMap<>();
public CloudTabletInvertedIndex() {
super();
diff --git a/fe/pom.xml b/fe/pom.xml
index 391d32998b7..c06131d3576 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -431,6 +431,7 @@ under the License.
<awssdk.version>2.29.52</awssdk.version>
<s3tables.catalog.version>0.1.4</s3tables.catalog.version>
<mockito.version>4.11.0</mockito.version>
+ <fastutil.version>8.5.18</fastutil.version>
</properties>
<profiles>
<profile>
@@ -1863,6 +1864,12 @@ under the License.
<artifactId>mockito-inline</artifactId>
<version>${mockito.version}</version>
</dependency>
+ <!-- fastutil for memory-efficient primitive collections -->
+ <dependency>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil-core</artifactId>
+ <version>${fastutil.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<dependencies>
@@ -1913,7 +1920,6 @@ under the License.
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
</dependency>
-
</dependencies>
<reporting>
<plugins>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
